mirror of
https://github.com/voideditor/void
synced 2026-05-24 09:58:23 +00:00
tool structure
This commit is contained in:
Andrew Pareles
parent
8591d06244
commit
131493b5e1
13 changed files with 530 additions and 466 deletions
|
|
@ -15,6 +15,7 @@ import { ILLMMessageService } from '../common/llmMessageService.js';
|
|||
import { IModelService } from '../../../../editor/common/services/model.js';
|
||||
import { chat_userMessage, chat_systemMessage } from './prompt/prompts.js';
|
||||
import { IToolsService, ToolName, voidTools } from '../common/toolsService.js';
|
||||
import { toLLMChatMessage } from '../common/llmMessageTypes.js';
|
||||
|
||||
// one of the square items that indicates a selection in a chat bubble (NOT a file, a Selection of text)
|
||||
export type CodeSelection = {
|
||||
|
|
@ -53,6 +54,7 @@ export type ChatMessage =
|
|||
}
|
||||
| {
|
||||
role: 'assistant';
|
||||
tool_calls?: { name: string, id: string, params: string }[];
|
||||
content: string | null; // content received from LLM - allowed to be '', will be replaced with (empty)
|
||||
displayContent: string | null; // content displayed to user (this is the same as content for now) - allowed to be '', will be ignored
|
||||
}
|
||||
|
|
@ -65,7 +67,7 @@ export type ChatMessage =
|
|||
role: 'tool';
|
||||
name: string; // internal use
|
||||
params: string | null; // internal use
|
||||
tool_use_id: string; // apis require this
|
||||
id: string; // apis require this tool use id
|
||||
content: string | null; // summary of the tool to the LLM
|
||||
displayContent: string | null; // text message of result
|
||||
}
|
||||
|
|
@ -296,82 +298,64 @@ class ChatThreadService extends Disposable implements IChatThreadService {
|
|||
|
||||
|
||||
// agent loop
|
||||
let shouldContinue = false
|
||||
do {
|
||||
shouldContinue = false
|
||||
const agentLoop = async () => {
|
||||
|
||||
let res_: () => void
|
||||
const awaitable = new Promise<void>((res, rej) => { res_ = res })
|
||||
let shouldContinue = false
|
||||
do {
|
||||
shouldContinue = false
|
||||
|
||||
const llmCancelToken = this._llmMessageService.sendLLMMessage({
|
||||
messagesType: 'chatMessages',
|
||||
useProviderFor: 'Ctrl+L',
|
||||
logging: { loggingName: `Agent` },
|
||||
messages: [
|
||||
{ role: 'system', content: chat_systemMessage },
|
||||
...this.getCurrentThread().messages.map(m => ({ ...m, content: m.content || '(empty model output)' })),
|
||||
],
|
||||
tools: [voidTools['read_file']], // TODO!!!!! make this change on agent | chat | search
|
||||
let res_: () => void
|
||||
const awaitable = new Promise<void>((res, rej) => { res_ = res })
|
||||
|
||||
onText: ({ fullText }) => {
|
||||
this._setStreamState(threadId, { messageSoFar: fullText })
|
||||
},
|
||||
onFinalMessage: async ({ fullText, tools }) => {
|
||||
if (tools.length === 0) {
|
||||
this._finishStreamingTextMessage(threadId, fullText)
|
||||
}
|
||||
else {
|
||||
for (const tool of tools) {
|
||||
if (!(tool.name in this._toolsService.toolFns)) {
|
||||
this._addMessageToThread(threadId, { role: 'tool', name: tool.name, params: tool.args, tool_use_id: tool.tool_use_id, content: `Error: This tool was not recognized, so it was not called.`, displayContent: `Error: tool not recognized.`, })
|
||||
}
|
||||
else {
|
||||
const toolName = tool.name as ToolName
|
||||
const toolResult = await this._toolsService.toolFns[toolName](tool.args)
|
||||
const string = this._toolsService.toolResultToString[toolName](toolResult as any) // typescript is so bad it doesn't even couple the type of ToolResult with the type of the function being called here
|
||||
this._addMessageToThread(threadId, { role: 'tool', name: tool.name, params: tool.args, tool_use_id: tool.tool_use_id, content: string, displayContent: string, })
|
||||
shouldContinue = true
|
||||
const llmCancelToken = this._llmMessageService.sendLLMMessage({
|
||||
messagesType: 'chatMessages',
|
||||
useProviderFor: 'Ctrl+L',
|
||||
logging: { loggingName: `Agent` },
|
||||
messages: [
|
||||
{ role: 'system', content: chat_systemMessage },
|
||||
...this.getCurrentThread().messages.map(m => (toLLMChatMessage(m))),
|
||||
],
|
||||
|
||||
// TODO!!!!! make this change on 'agent' | 'chat'
|
||||
tools: Object.keys(voidTools).map(toolName => voidTools[toolName as ToolName]),
|
||||
|
||||
onText: ({ fullText }) => {
|
||||
this._setStreamState(threadId, { messageSoFar: fullText })
|
||||
},
|
||||
onFinalMessage: async ({ fullText, tools }) => {
|
||||
if (tools.length === 0) {
|
||||
this._finishStreamingTextMessage(threadId, fullText)
|
||||
}
|
||||
else {
|
||||
for (const tool of tools) {
|
||||
if (!(tool.name in this._toolsService.toolFns)) {
|
||||
this._addMessageToThread(threadId, { role: 'tool', name: tool.name, params: tool.args, id: tool.id, content: `Error: This tool was not recognized, so it was not called.`, displayContent: `Error: tool not recognized.`, })
|
||||
}
|
||||
else {
|
||||
const toolName = tool.name as ToolName
|
||||
const toolResult = await this._toolsService.toolFns[toolName](tool.args)
|
||||
const string = this._toolsService.toolResultToString[toolName](toolResult as any) // typescript is so bad it doesn't even couple the type of ToolResult with the type of the function being called here
|
||||
this._addMessageToThread(threadId, { role: 'tool', name: tool.name, params: tool.args, id: tool.id, content: string, displayContent: string, })
|
||||
shouldContinue = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
res_()
|
||||
},
|
||||
onError: (error) => {
|
||||
this._finishStreamingTextMessage(threadId, this.streamState[threadId]?.messageSoFar ?? '', error)
|
||||
res_()
|
||||
},
|
||||
})
|
||||
if (llmCancelToken === null) return
|
||||
this._setStreamState(threadId, { streamingToken: llmCancelToken })
|
||||
res_()
|
||||
},
|
||||
onError: (error) => {
|
||||
this._finishStreamingTextMessage(threadId, this.streamState[threadId]?.messageSoFar ?? '', error)
|
||||
res_()
|
||||
},
|
||||
})
|
||||
if (llmCancelToken === null) break
|
||||
this._setStreamState(threadId, { streamingToken: llmCancelToken })
|
||||
|
||||
await awaitable
|
||||
await awaitable
|
||||
}
|
||||
while (shouldContinue);
|
||||
}
|
||||
while (shouldContinue);
|
||||
|
||||
|
||||
|
||||
|
||||
// const llmCancelToken = this._llmMessageService.sendLLMMessage({
|
||||
// messagesType: 'chatMessages',
|
||||
// logging: { loggingName: 'Chat' },
|
||||
// useProviderFor: 'Ctrl+L',
|
||||
// messages: [
|
||||
// { role: 'system', content: chat_systemMessage },
|
||||
// ...this.getCurrentThread().messages.map(m => ({ role: m.role, content: m.content || '(empty model output)' })),
|
||||
// ],
|
||||
// onText: ({ newText, fullText }) => {
|
||||
// this._setStreamState(threadId, { messageSoFar: fullText })
|
||||
// },
|
||||
// onFinalMessage: ({ fullText: content }) => {
|
||||
// this._finishStreaming(threadId, content)
|
||||
// },
|
||||
// onError: (error) => {
|
||||
// this._finishStreaming(threadId, this.streamState[threadId]?.messageSoFar ?? '', error)
|
||||
// },
|
||||
|
||||
// })
|
||||
// if (llmCancelToken === null) return
|
||||
// this._setStreamState(threadId, { streamingToken: llmCancelToken })
|
||||
agentLoop() // DO NOT AWAIT THIS, this fn should resolve when ready to clear inputs
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ import { Emitter } from '../../../../base/common/event.js';
|
|||
import { VOID_OPEN_SETTINGS_ACTION_ID } from './voidSettingsPane.js';
|
||||
import { ICommandService } from '../../../../platform/commands/common/commands.js';
|
||||
import { ILLMMessageService } from '../common/llmMessageService.js';
|
||||
import { LLMChatMessage, _InternalLLMChatMessage, errorDetails } from '../common/llmMessageTypes.js';
|
||||
import { LLMChatMessage, errorDetails } from '../common/llmMessageTypes.js';
|
||||
import { IMetricsService } from '../common/metricsService.js';
|
||||
import { VSReadFile } from './helpers/readFile.js';
|
||||
|
||||
|
|
@ -1178,13 +1178,16 @@ class EditCodeService extends Disposable implements IEditCodeService {
|
|||
private async _initializeSearchAndReplaceStream({ applyStr }: { applyStr: string }) {
|
||||
|
||||
|
||||
console.log('SEARCHREPLACE')
|
||||
const uri_ = this._getActiveEditorURI()
|
||||
if (!uri_) return
|
||||
const uri = uri_
|
||||
|
||||
console.log('/* AAAA */')
|
||||
// generate search/replace block text
|
||||
const fileContents = await VSReadFile(this._modelService, uri)
|
||||
if (fileContents === null) return
|
||||
console.log('/* BBB*/')
|
||||
|
||||
|
||||
|
||||
|
|
@ -1236,9 +1239,7 @@ class EditCodeService extends Disposable implements IEditCodeService {
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
// TODO turn this into a service and provide it
|
||||
// TODO!!! turn this into a service and provide it
|
||||
streamRequestIdRef.current = this._llmMessageService.sendLLMMessage({
|
||||
messagesType: 'chatMessages',
|
||||
useProviderFor: 'Apply',
|
||||
|
|
@ -1304,6 +1305,8 @@ class EditCodeService extends Disposable implements IEditCodeService {
|
|||
this._refreshStylesAndDiffsInURI(uri)
|
||||
},
|
||||
onFinalMessage: async ({ fullText }) => {
|
||||
console.log('/* ONFIN */', fullText)
|
||||
|
||||
// 1. wait 500ms and fix lint errors - call lint error workflow
|
||||
// (update react state to say "Fixing errors")
|
||||
const blocks = extractSearchReplaceBlocks(fullText)
|
||||
|
|
@ -1322,6 +1325,8 @@ class EditCodeService extends Disposable implements IEditCodeService {
|
|||
onDone(false)
|
||||
},
|
||||
onError: (e) => {
|
||||
console.log('/* ERRRRRR */')
|
||||
|
||||
console.log('ERROR', e);
|
||||
onDone(true)
|
||||
},
|
||||
|
|
|
|||
|
|
@ -395,7 +395,16 @@ export const AIInstructionsBox = () => {
|
|||
|
||||
export const FeaturesTab = () => {
|
||||
return <>
|
||||
<h2 className={`text-3xl mb-2`}>Local Providers</h2>
|
||||
<h2 className={`text-3xl mb-2`}>Models</h2>
|
||||
<ErrorBoundary>
|
||||
<AutoRefreshToggle />
|
||||
<RefreshableModels />
|
||||
<ModelDump />
|
||||
<AddModelMenuFull />
|
||||
</ErrorBoundary>
|
||||
|
||||
|
||||
<h2 className={`text-3xl mb-2 mt-12`}>Local Providers</h2>
|
||||
{/* <h3 className={`opacity-50 mb-2`}>{`Keep your data private by hosting AI locally on your computer.`}</h3> */}
|
||||
{/* <h3 className={`opacity-50 mb-2`}>{`Instructions:`}</h3> */}
|
||||
{/* <h3 className={`mb-2`}>{`Void can access any model that you host locally. We automatically detect your local models by default.`}</h3> */}
|
||||
|
|
@ -420,13 +429,20 @@ export const FeaturesTab = () => {
|
|||
<VoidProviderSettings providerNames={nonlocalProviderNames} />
|
||||
</ErrorBoundary>
|
||||
|
||||
<h2 className={`text-3xl mb-2 mt-12`}>Models</h2>
|
||||
|
||||
|
||||
<h2 className={`text-3xl mb-2 mt-12`}>Feature Options</h2>
|
||||
<ErrorBoundary>
|
||||
<AutoRefreshToggle />
|
||||
<RefreshableModels />
|
||||
<ModelDump />
|
||||
<AddModelMenuFull />
|
||||
{featureNames.map(featureName =>
|
||||
<div key={featureName}
|
||||
className='mb-2'
|
||||
>
|
||||
<h4 className={`text-void-fg-3`}>{displayInfoOfFeatureName(featureName)}</h4>
|
||||
<ModelDropdown featureName={featureName} />
|
||||
</div>
|
||||
)}
|
||||
</ErrorBoundary>
|
||||
|
||||
</>
|
||||
}
|
||||
|
||||
|
|
@ -588,17 +604,6 @@ const GeneralTab = () => {
|
|||
<AIInstructionsBox />
|
||||
</div>
|
||||
|
||||
<div className='mt-12'>
|
||||
<h2 className={`text-3xl mb-2`}>Model Selection</h2>
|
||||
{featureNames.map(featureName =>
|
||||
<div key={featureName}
|
||||
className='mb-2'
|
||||
>
|
||||
<h4 className={`text-void-fg-3`}>{displayInfoOfFeatureName(featureName)}</h4>
|
||||
<ModelDropdown featureName={featureName} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
</>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import { ChatMessage } from '../browser/chatThreadService.js'
|
||||
import { InternalToolInfo } from './toolsService.js'
|
||||
import { FeatureName, ProviderName, SettingsOfProvider } from './voidSettingsTypes.js'
|
||||
|
||||
|
|
@ -22,7 +23,7 @@ export const errorDetails = (fullError: Error | null): string | null => {
|
|||
}
|
||||
|
||||
export type OnText = (p: { newText: string, fullText: string }) => void
|
||||
export type OnFinalMessage = (p: { fullText: string, tools: { name: string, args: string, tool_use_id: string, }[] }) => void
|
||||
export type OnFinalMessage = (p: { fullText: string, tools: { name: string, args: string, id: string, }[] }) => void // id is tool_use_id
|
||||
export type OnError = (p: { message: string, fullError: Error | null }) => void
|
||||
export type AbortRef = { current: (() => void) | null }
|
||||
|
||||
|
|
@ -30,20 +31,32 @@ export type LLMChatMessage = {
|
|||
role: 'system' | 'user';
|
||||
content: string;
|
||||
} | {
|
||||
role: 'tool';
|
||||
tool_use_id: string;
|
||||
role: 'assistant',
|
||||
tool_calls?: { name: string, id: string, params: string }[];
|
||||
content: string;
|
||||
} | {
|
||||
role: 'assistant',
|
||||
tool_calls?: { name: string, tool_use_id: string, params: string }[];
|
||||
role: 'tool';
|
||||
id: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export const toLLMChatMessage = (c: ChatMessage): LLMChatMessage => {
|
||||
if (c.role === 'system' || c.role === 'user') {
|
||||
return { role: c.role, content: c.content ?? '(empty)' }
|
||||
}
|
||||
else if (c.role === 'assistant')
|
||||
return { role: c.role, tool_calls: c.tool_calls, content: c.content ?? '(empty model output)' }
|
||||
else if (c.role === 'tool')
|
||||
return { role: c.role, id: c.id, content: c.content ?? '(empty output)' }
|
||||
else {
|
||||
throw 1
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export type _InternalLLMChatMessage = {
|
||||
role: any;
|
||||
tool_use_id?: any;
|
||||
id?: any;
|
||||
content: string;
|
||||
}
|
||||
|
||||
|
|
@ -112,7 +125,7 @@ export type _InternalSendLLMChatMessageFnType = (
|
|||
|
||||
tools?: InternalToolInfo[],
|
||||
|
||||
messages: _InternalLLMChatMessage[];
|
||||
messages: LLMChatMessage[];
|
||||
}
|
||||
) => void
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { registerSingleton, InstantiationType } from '../../../../platform/insta
|
|||
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
|
||||
import { IStorageService, StorageScope, StorageTarget } from '../../../../platform/storage/common/storage.js';
|
||||
import { IMetricsService } from './metricsService.js';
|
||||
import { defaultSettingsOfProvider, FeatureName, ProviderName, ModelSelectionOfFeature, SettingsOfProvider, SettingName, providerNames, ModelSelection, modelSelectionsEqual, featureNames, VoidModelInfo, GlobalSettings, GlobalSettingName, defaultGlobalSettings, displayInfoOfProviderName, defaultProviderSettings, developerInfoOfModelName, modelInfoOfAutodetectedModelNames } from './voidSettingsTypes.js';
|
||||
import { defaultSettingsOfProvider, FeatureName, ProviderName, ModelSelectionOfFeature, SettingsOfProvider, SettingName, providerNames, ModelSelection, modelSelectionsEqual, featureNames, VoidModelInfo, GlobalSettings, GlobalSettingName, defaultGlobalSettings, defaultProviderSettings, developerInfoOfModelName, modelInfoOfAutodetectedModelNames } from './voidSettingsTypes.js';
|
||||
|
||||
|
||||
const STORAGE_KEY = 'void.settingsServiceStorage'
|
||||
|
|
@ -89,7 +89,7 @@ const _updatedValidatedState = (state: Omit<VoidSettingsState, '_modelOptions'>)
|
|||
// update model options
|
||||
let newModelOptions: ModelOption[] = []
|
||||
for (const providerName of providerNames) {
|
||||
const providerTitle = displayInfoOfProviderName(providerName).title.toLowerCase() // looks better lowercase, best practice to not use raw providerName
|
||||
const providerTitle = providerName // displayInfoOfProviderName(providerName).title.toLowerCase() // looks better lowercase, best practice to not use raw providerName
|
||||
if (!newSettingsOfProvider[providerName]._didFillInProviderSettings) continue // if disabled, don't display model options
|
||||
for (const { modelName, isHidden } of newSettingsOfProvider[providerName].models) {
|
||||
if (isHidden) continue
|
||||
|
|
|
|||
|
|
@ -25,9 +25,7 @@ export type DeveloperInfoAtModel = {
|
|||
}
|
||||
|
||||
export type DeveloperInfoAtProvider = {
|
||||
separateSystemMessage?: boolean;
|
||||
toolsGoInRole?: boolean; // whether to do {role:'tool'} or {role:'user' tool:...}
|
||||
modelOverrides?: Partial<DeveloperInfoAtModel>; // any overrides for models that a provider might have (e.g. if a provider always supports tool use, even if we don't recognize the model we can set tools to true)
|
||||
overrideSettingsForAllModels?: Partial<DeveloperInfoAtModel>; // any overrides for models that a provider might have (e.g. if a provider always supports tool use, even if we don't recognize the model we can set tools to true)
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -99,37 +97,34 @@ export function recognizedModelOfModelName(modelName: string): RecognizedModelNa
|
|||
|
||||
const developerInfoAtProvider: { [providerName in ProviderName]: DeveloperInfoAtProvider } = {
|
||||
'anthropic': {
|
||||
separateSystemMessage: true,
|
||||
toolsGoInRole: false,
|
||||
modelOverrides: {
|
||||
overrideSettingsForAllModels: {
|
||||
supportsSystemMessage: 'system',
|
||||
supportsTools: true,
|
||||
supportsAutocompleteFIM: false,
|
||||
supportsStreaming: true,
|
||||
}
|
||||
},
|
||||
'deepseek': {
|
||||
separateSystemMessage: true,
|
||||
},
|
||||
'openAI': {
|
||||
separateSystemMessage: false,
|
||||
toolsGoInRole: true,
|
||||
},
|
||||
'gemini': {
|
||||
separateSystemMessage: true,
|
||||
toolsGoInRole: false
|
||||
},
|
||||
'mistral': {
|
||||
separateSystemMessage: true,
|
||||
},
|
||||
'groq': {
|
||||
separateSystemMessage: true,
|
||||
overrideSettingsForAllModels: {
|
||||
supportsSystemMessage: false,
|
||||
supportsTools: false,
|
||||
supportsAutocompleteFIM: false,
|
||||
supportsStreaming: true,
|
||||
}
|
||||
},
|
||||
'ollama': {
|
||||
separateSystemMessage: false,
|
||||
},
|
||||
'openRouter': {
|
||||
separateSystemMessage: true,
|
||||
},
|
||||
'openAICompatible': {
|
||||
separateSystemMessage: true,
|
||||
},
|
||||
'openAI': {
|
||||
},
|
||||
'gemini': {
|
||||
},
|
||||
'mistral': {
|
||||
},
|
||||
'groq': {
|
||||
},
|
||||
}
|
||||
export const developerInfoOfProviderName = (providerName: ProviderName): Partial<DeveloperInfoAtProvider> => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
// /*--------------------------------------------------------------------------------------
|
||||
// * Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
// * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
// *--------------------------------------------------------------------------------------*/
|
||||
|
||||
// import Groq from 'groq-sdk';
|
||||
// import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js';
|
||||
|
||||
// // Groq
|
||||
// export const sendGroqChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
// let fullText = '';
|
||||
|
||||
// const thisConfig = settingsOfProvider.groq
|
||||
|
||||
// const groq = new Groq({
|
||||
// apiKey: thisConfig.apiKey,
|
||||
// dangerouslyAllowBrowser: true
|
||||
// });
|
||||
|
||||
// await groq.chat.completions
|
||||
// .create({
|
||||
// messages: messages,
|
||||
// model: modelName,
|
||||
// stream: true,
|
||||
// })
|
||||
// .then(async response => {
|
||||
// _setAborter(() => response.controller.abort())
|
||||
// // when receive text
|
||||
// for await (const chunk of response) {
|
||||
// const newText = chunk.choices[0]?.delta?.content || '';
|
||||
// fullText += newText;
|
||||
// onText({ newText, fullText });
|
||||
// }
|
||||
|
||||
// onFinalMessage({ fullText, tools: [] });
|
||||
// })
|
||||
// .catch(error => {
|
||||
// onError({ message: error + '', fullError: error });
|
||||
// })
|
||||
|
||||
|
||||
// };
|
||||
|
||||
|
||||
|
||||
// /*--------------------------------------------------------------------------------------
|
||||
// * Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
// * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
// *--------------------------------------------------------------------------------------*/
|
||||
|
||||
// import { Mistral } from '@mistralai/mistralai';
|
||||
// import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js';
|
||||
|
||||
// // Mistral
|
||||
// export const sendMistralChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
// let fullText = '';
|
||||
|
||||
// const thisConfig = settingsOfProvider.mistral;
|
||||
|
||||
// const mistral = new Mistral({
|
||||
// apiKey: thisConfig.apiKey,
|
||||
// })
|
||||
|
||||
// await mistral.chat
|
||||
// .stream({
|
||||
// messages: messages,
|
||||
// model: modelName,
|
||||
// stream: true,
|
||||
// })
|
||||
// .then(async response => {
|
||||
// // Mistral has a really nonstandard API - no interrupt and weird stream types
|
||||
// _setAborter(() => { console.log('Mistral does not support interrupts! Further messages will just be ignored.') });
|
||||
// // when receive text
|
||||
// for await (const chunk of response) {
|
||||
// const c = chunk.data.choices[0].delta.content || ''
|
||||
// const newText = (
|
||||
// typeof c === 'string' ? c
|
||||
// : c?.map(c => c.type === 'text' ? c.text : c.type).join('\n')
|
||||
// )
|
||||
// fullText += newText;
|
||||
// onText({ newText, fullText });
|
||||
// }
|
||||
|
||||
// onFinalMessage({ fullText, tools: [] });
|
||||
// })
|
||||
// .catch(error => {
|
||||
// onError({ message: error + '', fullError: error });
|
||||
// })
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
import { _InternalLLMChatMessage, LLMChatMessage } from '../../common/llmMessageTypes.js';
|
||||
import { DeveloperInfoAtModel, developerInfoOfModelName, developerInfoOfProviderName, ProviderName } from '../../common/voidSettingsTypes.js';
|
||||
|
||||
|
||||
// no matter whether the model supports a system message or not (or what format it supports), add it in some way
|
||||
// also take into account tools if the model doesn't support tool use
|
||||
export const addSystemMessageAndToolSupport = (modelName: string, providerName: ProviderName, messages_: LLMChatMessage[], { separateSystemMessage }: { separateSystemMessage: boolean }): { separateSystemMessageStr?: string, messages: _InternalLLMChatMessage[], devInfo: DeveloperInfoAtModel } => {
|
||||
|
||||
const messages: _InternalLLMChatMessage[] = messages_.map(m => ({ ...m, content: m.content.trim(), }))
|
||||
|
||||
const { overrideSettingsForAllModels } = developerInfoOfProviderName(providerName)
|
||||
const devInfo = developerInfoOfModelName(modelName, overrideSettingsForAllModels)
|
||||
const { supportsSystemMessage } = devInfo
|
||||
|
||||
// 1. SYSTEM MESSAGE
|
||||
// find system messages and concatenate them
|
||||
let systemMessageStr = messages
|
||||
.filter(msg => msg.role === 'system')
|
||||
.map(msg => msg.content)
|
||||
.join('\n') || undefined;
|
||||
|
||||
let separateSystemMessageStr: string | undefined = undefined
|
||||
|
||||
// remove all system messages
|
||||
const newMessages: _InternalLLMChatMessage[] = messages.filter(msg => msg.role !== 'system')
|
||||
|
||||
|
||||
// if (!supportsTools) {
|
||||
// if (!systemMessageStr) systemMessageStr = ''
|
||||
// systemMessageStr += '' // TODO!!! add tool use system message here
|
||||
// }
|
||||
|
||||
|
||||
if (systemMessageStr) {
|
||||
// if supports system message
|
||||
if (supportsSystemMessage) {
|
||||
if (separateSystemMessage)
|
||||
separateSystemMessageStr = systemMessageStr
|
||||
else {
|
||||
newMessages.unshift({ role: supportsSystemMessage, content: systemMessageStr }) // add new first message
|
||||
}
|
||||
}
|
||||
// if does not support system message
|
||||
else {
|
||||
if (supportsSystemMessage) {
|
||||
if (newMessages.length === 0)
|
||||
newMessages.push({ role: 'user', content: systemMessageStr })
|
||||
// add system mesasges to first message (should be a user message)
|
||||
else {
|
||||
const newFirstMessage = {
|
||||
role: newMessages[0].role,
|
||||
content: (''
|
||||
+ '<SYSTEM_MESSAGE>\n'
|
||||
+ systemMessageStr
|
||||
+ '\n'
|
||||
+ '</SYSTEM_MESSAGE>\n'
|
||||
+ newMessages[0].content
|
||||
)
|
||||
}
|
||||
newMessages.splice(0, 1) // delete first message
|
||||
newMessages.unshift(newFirstMessage) // add new first message
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return {
|
||||
separateSystemMessageStr,
|
||||
messages: newMessages,
|
||||
devInfo,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// const { maxTokens, supportsTools, supportsAutocompleteFIM, supportsStreaming, } = developerInfoOfModelName(recognizedModel)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// let index = 0;
|
||||
// while (index < newMessages.length) {
|
||||
|
||||
// merge tool with the previous assistant and the following user message
|
||||
|
||||
// take prev message and add
|
||||
/*
|
||||
openai assistant message will have: https://platform.openai.com/docs/guides/function-calling#function-calling-steps
|
||||
"tool_calls":[
|
||||
{
|
||||
"id": "call_12345xyz",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": "{\"latitude\":48.8566,\"longitude\":2.3522}"
|
||||
}
|
||||
}]
|
||||
|
||||
openai user response will be:
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tool_call.id,
|
||||
"content": str(result)
|
||||
}
|
||||
|
||||
anthropic assistant message will have: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#tool-use-examples
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "<thinking>I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA.</thinking>"
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01A09q90qw90lq917835lq9",
|
||||
"name": "get_weather",
|
||||
"input": {"location": "San Francisco, CA", "unit": "celsius"}
|
||||
}
|
||||
]
|
||||
|
||||
anthropic user message response will be:
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01A09q90qw90lq917835lq9",
|
||||
"content": "15 degrees"
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/*
|
||||
|
||||
|
||||
ACCORDING TO 4o: gemini: similar to openai, but function_call, and only 1 call per message (no id because only 1 message)
|
||||
gemini request: {
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"function_call": {
|
||||
"name": "get_weather",
|
||||
"arguments": {
|
||||
"latitude": 48.8566,
|
||||
"longitude": 2.3522
|
||||
}
|
||||
}
|
||||
}
|
||||
gemini response:
|
||||
{
|
||||
"role": "assistant",
|
||||
"function_response": {
|
||||
"name": "get_weather",
|
||||
"response": {
|
||||
"temperature": "15°C",
|
||||
"condition": "Cloudy"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+ anthropic
|
||||
|
||||
+ openai-compat (4)
|
||||
+ gemini
|
||||
|
||||
ollama
|
||||
|
||||
|
||||
mistral: same as openai
|
||||
|
||||
*/
|
||||
|
|
@ -7,6 +7,7 @@ import Anthropic from '@anthropic-ai/sdk';
|
|||
import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js';
|
||||
import { anthropicMaxPossibleTokens } from '../../common/voidSettingsTypes.js';
|
||||
import { InternalToolInfo } from '../../common/toolsService.js';
|
||||
import { addSystemMessageAndToolSupport } from './addSupport.js';
|
||||
|
||||
|
||||
|
||||
|
|
@ -28,7 +29,7 @@ export const toAnthropicTool = (toolInfo: InternalToolInfo) => {
|
|||
|
||||
|
||||
|
||||
export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, tools }) => {
|
||||
export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages: messages_, providerName, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, tools: tools_ }) => {
|
||||
|
||||
const thisConfig = settingsOfProvider.anthropic
|
||||
|
||||
|
|
@ -38,15 +39,19 @@ export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages,
|
|||
return
|
||||
}
|
||||
|
||||
const { messages, separateSystemMessageStr, devInfo } = addSystemMessageAndToolSupport(modelName, providerName, messages_, { separateSystemMessage: true })
|
||||
|
||||
const anthropic = new Anthropic({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true });
|
||||
|
||||
const tools = devInfo?.supportsTools && (tools_?.length ?? 0) !== 0 ? tools_?.map(tool => toAnthropicTool(tool)) : undefined
|
||||
|
||||
const stream = anthropic.messages.stream({
|
||||
// system: systemMessage,
|
||||
system: separateSystemMessageStr,
|
||||
messages: messages,
|
||||
model: modelName,
|
||||
max_tokens: maxTokens,
|
||||
tools: tools?.map(tool => toAnthropicTool(tool)),
|
||||
tool_choice: { type: 'auto', disable_parallel_tool_use: true } // one tool use at a time
|
||||
tools: tools,
|
||||
tool_choice: tools ? { type: 'auto', disable_parallel_tool_use: true } : undefined // one tool use at a time
|
||||
})
|
||||
|
||||
|
||||
|
|
@ -78,9 +83,9 @@ export const sendAnthropicChat: _InternalSendLLMChatMessageFnType = ({ messages,
|
|||
stream.on('finalMessage', (response) => {
|
||||
// stringify the response's content
|
||||
const content = response.content.map(c => c.type === 'text' ? c.text : '').join('\n\n')
|
||||
const tools = response.content.map(c => c.type === 'tool_use' ? { name: c.name, args: JSON.stringify(c.input), tool_use_id: c.id } : null).filter(c => !!c)
|
||||
// const tools = response.content.map(c => c.type === 'tool_use' ? { name: c.name, args: JSON.stringify(c.input), tool_use_id: c.id } : null).filter(c => !!c)
|
||||
|
||||
onFinalMessage({ fullText: content, tools })
|
||||
onFinalMessage({ fullText: content, tools: [] })
|
||||
})
|
||||
|
||||
stream.on('error', (error) => {
|
||||
|
|
|
|||
|
|
@ -1,43 +0,0 @@
|
|||
/*--------------------------------------------------------------------------------------
|
||||
* Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import { Content, GoogleGenerativeAI } from '@google/generative-ai';
|
||||
import { _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js';
|
||||
|
||||
// Gemini
|
||||
export const sendGeminiChat: _InternalSendLLMChatMessageFnType = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
|
||||
let fullText = ''
|
||||
|
||||
const thisConfig = settingsOfProvider.gemini
|
||||
|
||||
const genAI = new GoogleGenerativeAI(thisConfig.apiKey);
|
||||
const model = genAI.getGenerativeModel({ model: modelName });
|
||||
|
||||
// Convert messages to Gemini format
|
||||
const geminiMessages: Content[] = messages
|
||||
.map((msg, i) => ({
|
||||
parts: [{ text: msg.content }],
|
||||
role: msg.role === 'assistant' ? 'model' : 'user'
|
||||
}))
|
||||
|
||||
model.generateContentStream({
|
||||
// systemInstruction: systemMessage,
|
||||
contents: geminiMessages,
|
||||
})
|
||||
.then(async response => {
|
||||
_setAborter(() => response.stream.return(fullText))
|
||||
|
||||
for await (const chunk of response.stream) {
|
||||
const newText = chunk.text();
|
||||
fullText += newText;
|
||||
onText({ newText, fullText });
|
||||
}
|
||||
onFinalMessage({ fullText, tools: [] });
|
||||
})
|
||||
.catch((error) => {
|
||||
onError({ message: error + '', fullError: error })
|
||||
})
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
/*--------------------------------------------------------------------------------------
|
||||
* Copyright 2025 Glass Devtools, Inc. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
|
|
@ -38,84 +39,86 @@ export const ollamaList: _InternalModelListFnType<OllamaModelResponse> = async (
|
|||
}
|
||||
|
||||
|
||||
export const sendOllamaFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
|
||||
const thisConfig = settingsOfProvider.ollama
|
||||
// if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`)
|
||||
|
||||
let fullText = ''
|
||||
|
||||
const ollama = new Ollama({ host: thisConfig.endpoint })
|
||||
|
||||
ollama.generate({
|
||||
model: modelName,
|
||||
prompt: messages.prefix,
|
||||
suffix: messages.suffix,
|
||||
options: {
|
||||
stop: messages.stopTokens,
|
||||
num_predict: 300, // max tokens
|
||||
// repeat_penalty: 1,
|
||||
},
|
||||
raw: true,
|
||||
stream: true,
|
||||
})
|
||||
.then(async stream => {
|
||||
_setAborter(() => stream.abort())
|
||||
// iterate through the stream
|
||||
for await (const chunk of stream) {
|
||||
const newText = chunk.response;
|
||||
fullText += newText;
|
||||
onText({ newText, fullText });
|
||||
}
|
||||
onFinalMessage({ fullText, tools: [] });
|
||||
})
|
||||
// when error/fail
|
||||
.catch((error) => {
|
||||
onError({ message: error + '', fullError: error })
|
||||
})
|
||||
};
|
||||
|
||||
|
||||
// Ollama
|
||||
export const sendOllamaChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
// export const sendOllamaFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
|
||||
const thisConfig = settingsOfProvider.ollama
|
||||
// if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`)
|
||||
// const thisConfig = settingsOfProvider.ollama
|
||||
// // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
// if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`)
|
||||
|
||||
let fullText = ''
|
||||
// let fullText = ''
|
||||
|
||||
const ollama = new Ollama({ host: thisConfig.endpoint })
|
||||
// const ollama = new Ollama({ host: thisConfig.endpoint })
|
||||
|
||||
ollama.chat({
|
||||
model: modelName,
|
||||
messages: messages,
|
||||
stream: true,
|
||||
// options: { num_predict: parseMaxTokensStr(thisConfig.maxTokens) } // this is max_tokens
|
||||
})
|
||||
.then(async stream => {
|
||||
_setAborter(() => stream.abort())
|
||||
// iterate through the stream
|
||||
for await (const chunk of stream) {
|
||||
const newText = chunk.message.content;
|
||||
// ollama.generate({
|
||||
// model: modelName,
|
||||
// prompt: messages.prefix,
|
||||
// suffix: messages.suffix,
|
||||
// options: {
|
||||
// stop: messages.stopTokens,
|
||||
// num_predict: 300, // max tokens
|
||||
// // repeat_penalty: 1,
|
||||
// },
|
||||
// raw: true,
|
||||
// stream: true,
|
||||
// })
|
||||
// .then(async stream => {
|
||||
// _setAborter(() => stream.abort())
|
||||
// // iterate through the stream
|
||||
// for await (const chunk of stream) {
|
||||
// const newText = chunk.response;
|
||||
// fullText += newText;
|
||||
// onText({ newText, fullText });
|
||||
// }
|
||||
// onFinalMessage({ fullText, tools: [] });
|
||||
// })
|
||||
// // when error/fail
|
||||
// .catch((error) => {
|
||||
// onError({ message: error + '', fullError: error })
|
||||
// })
|
||||
// };
|
||||
|
||||
// chunk.message.tool_calls[0].function.arguments
|
||||
|
||||
fullText += newText;
|
||||
onText({ newText, fullText });
|
||||
}
|
||||
// // Ollama
|
||||
// export const sendOllamaChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter }) => {
|
||||
|
||||
onFinalMessage({ fullText, tools: [] });
|
||||
// const thisConfig = settingsOfProvider.ollama
|
||||
// // if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
// if (!thisConfig.endpoint) throw new Error(`Ollama Endpoint was empty (please enter ${defaultProviderSettings.ollama.endpoint} if you want the default).`)
|
||||
|
||||
})
|
||||
// when error/fail
|
||||
.catch((error) => {
|
||||
onError({ message: error + '', fullError: error })
|
||||
})
|
||||
// let fullText = ''
|
||||
|
||||
};
|
||||
// const ollama = new Ollama({ host: thisConfig.endpoint })
|
||||
|
||||
// ollama.chat({
|
||||
// model: modelName,
|
||||
// messages: messages,
|
||||
// stream: true,
|
||||
// // options: { num_predict: parseMaxTokensStr(thisConfig.maxTokens) } // this is max_tokens
|
||||
// })
|
||||
// .then(async stream => {
|
||||
// _setAborter(() => stream.abort())
|
||||
// // iterate through the stream
|
||||
// for await (const chunk of stream) {
|
||||
// const newText = chunk.message.content;
|
||||
|
||||
// // chunk.message.tool_calls[0].function.arguments
|
||||
|
||||
// fullText += newText;
|
||||
// onText({ newText, fullText });
|
||||
// }
|
||||
|
||||
// onFinalMessage({ fullText, tools: [] });
|
||||
|
||||
// })
|
||||
// // when error/fail
|
||||
// .catch((error) => {
|
||||
// onError({ message: error + '', fullError: error })
|
||||
// })
|
||||
|
||||
// };
|
||||
|
||||
|
||||
|
||||
// ['codestral', 'qwen2.5-coder', 'qwen2.5-coder:0.5b', 'qwen2.5-coder:1.5b', 'qwen2.5-coder:3b', 'qwen2.5-coder:7b', 'qwen2.5-coder:14b', 'qwen2.5-coder:32b', 'codegemma', 'codegemma:2b', 'codegemma:7b', 'codellama', 'codellama:7b', 'codellama:13b', 'codellama:34b', 'codellama:70b', 'codellama:code', 'codellama:python', 'command-r', 'command-r:35b', 'command-r-plus', 'command-r-plus:104b', 'deepseek-coder-v2', 'deepseek-coder-v2:16b', 'deepseek-coder-v2:236b', 'falcon2', 'falcon2:11b', 'firefunction-v2', 'firefunction-v2:70b', 'gemma', 'gemma:2b', 'gemma:7b', 'gemma2', 'gemma2:2b', 'gemma2:9b', 'gemma2:27b', 'llama2', 'llama2:7b', 'llama2:13b', 'llama2:70b', 'llama3', 'llama3:8b', 'llama3:70b', 'llama3-chatqa', 'llama3-chatqa:8b', 'llama3-chatqa:70b', 'llama3-gradient', 'llama3-gradient:8b', 'llama3-gradient:70b', 'llama3.1', 'llama3.1:8b', 'llama3.1:70b', 'llama3.1:405b', 'llava', 'llava:7b', 'llava:13b', 'llava:34b', 'llava-llama3', 'llava-llama3:8b', 'llava-phi3', 'llava-phi3:3.8b', 'mistral', 'mistral:7b', 'mistral-large', 'mistral-large:123b', 'mistral-nemo', 'mistral-nemo:12b', 'mixtral', 'mixtral:8x7b', 'mixtral:8x22b', 'moondream', 'moondream:1.8b', 'openhermes', 'openhermes:v2.5', 'phi3', 'phi3:3.8b', 'phi3:14b', 'phi3.5', 'phi3.5:3.8b', 'qwen', 'qwen:7b', 'qwen:14b', 'qwen:32b', 'qwen:72b', 'qwen:110b', 'qwen2', 'qwen2:0.5b', 'qwen2:1.5b', 'qwen2:7b', 'qwen2:72b', 'smollm', 'smollm:135m', 'smollm:360m', 'smollm:1.7b',]
|
||||
// // ['codestral', 'qwen2.5-coder', 'qwen2.5-coder:0.5b', 'qwen2.5-coder:1.5b', 'qwen2.5-coder:3b', 'qwen2.5-coder:7b', 'qwen2.5-coder:14b', 'qwen2.5-coder:32b', 'codegemma', 'codegemma:2b', 'codegemma:7b', 'codellama', 'codellama:7b', 'codellama:13b', 'codellama:34b', 'codellama:70b', 'codellama:code', 'codellama:python', 'command-r', 'command-r:35b', 'command-r-plus', 'command-r-plus:104b', 'deepseek-coder-v2', 'deepseek-coder-v2:16b', 'deepseek-coder-v2:236b', 'falcon2', 'falcon2:11b', 'firefunction-v2', 'firefunction-v2:70b', 'gemma', 'gemma:2b', 'gemma:7b', 'gemma2', 'gemma2:2b', 'gemma2:9b', 'gemma2:27b', 'llama2', 'llama2:7b', 'llama2:13b', 'llama2:70b', 'llama3', 'llama3:8b', 'llama3:70b', 'llama3-chatqa', 'llama3-chatqa:8b', 'llama3-chatqa:70b', 'llama3-gradient', 'llama3-gradient:8b', 'llama3-gradient:70b', 'llama3.1', 'llama3.1:8b', 'llama3.1:70b', 'llama3.1:405b', 'llava', 'llava:7b', 'llava:13b', 'llava:34b', 'llava-llama3', 'llava-llama3:8b', 'llava-phi3', 'llava-phi3:3.8b', 'mistral', 'mistral:7b', 'mistral-large', 'mistral-large:123b', 'mistral-nemo', 'mistral-nemo:12b', 'mixtral', 'mixtral:8x7b', 'mixtral:8x22b', 'moondream', 'moondream:1.8b', 'openhermes', 'openhermes:v2.5', 'phi3', 'phi3:3.8b', 'phi3:14b', 'phi3.5', 'phi3.5:3.8b', 'qwen', 'qwen:7b', 'qwen:14b', 'qwen:32b', 'qwen:72b', 'qwen:110b', 'qwen2', 'qwen2:0.5b', 'qwen2:1.5b', 'qwen2:7b', 'qwen2:72b', 'smollm', 'smollm:135m', 'smollm:360m', 'smollm:1.7b',]
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import OpenAI from 'openai';
|
|||
import { _InternalModelListFnType, _InternalSendLLMFIMMessageFnType, _InternalSendLLMChatMessageFnType } from '../../common/llmMessageTypes.js';
|
||||
import { Model } from 'openai/resources/models.js';
|
||||
import { InternalToolInfo } from '../../common/toolsService.js';
|
||||
import { addSystemMessageAndToolSupport } from './addSupport.js';
|
||||
// import { parseMaxTokensStr } from './util.js';
|
||||
|
||||
|
||||
|
|
@ -38,11 +39,19 @@ type NewParams = Pick<Parameters<_InternalSendLLMChatMessageFnType>[0] & Paramet
|
|||
const newOpenAI = ({ settingsOfProvider, providerName }: NewParams) => {
|
||||
|
||||
if (providerName === 'openAI') {
|
||||
const thisConfig = settingsOfProvider.openAI
|
||||
return new OpenAI({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true });
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true
|
||||
})
|
||||
}
|
||||
else if (providerName === 'ollama') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'openRouter') {
|
||||
const thisConfig = settingsOfProvider.openRouter
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://openrouter.ai/api/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
defaultHeaders: {
|
||||
|
|
@ -51,33 +60,38 @@ const newOpenAI = ({ settingsOfProvider, providerName }: NewParams) => {
|
|||
},
|
||||
})
|
||||
}
|
||||
else if (providerName === 'gemini') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'deepseek') {
|
||||
const thisConfig = settingsOfProvider.deepseek
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
|
||||
}
|
||||
else if (providerName === 'openAICompatible') {
|
||||
const thisConfig = settingsOfProvider.openAICompatible
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true
|
||||
baseURL: thisConfig.endpoint, apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'mistral') {
|
||||
const thisConfig = settingsOfProvider.mistral
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: 'https://api.mistral.ai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else if (providerName === 'groq') {
|
||||
const thisConfig = settingsOfProvider.groq
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
baseURL: '"https://api.groq.com/openai/v1"', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
baseURL: 'https://api.groq.com/openai/v1', apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true,
|
||||
})
|
||||
}
|
||||
else {
|
||||
console.error(`sendOpenAIMsg: invalid providerName: ${providerName}`)
|
||||
console.error(`sendOpenAICompatibleMsg: invalid providerName: ${providerName}`)
|
||||
throw new Error(`providerName was invalid: ${providerName}`)
|
||||
}
|
||||
}
|
||||
|
|
@ -130,10 +144,14 @@ export const sendOpenAIFIM: _InternalSendLLMFIMMessageFnType = ({ messages, onTe
|
|||
|
||||
|
||||
// OpenAI, OpenRouter, OpenAICompatible
|
||||
export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools }) => {
|
||||
export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools: tools_ }) => {
|
||||
|
||||
let fullText = ''
|
||||
const toolCallOfIndex: { [index: string]: { name: string, args: string } } = {}
|
||||
const toolCallOfIndex: { [index: string]: { name: string, args: string, id: string } } = {}
|
||||
|
||||
const { messages, devInfo } = addSystemMessageAndToolSupport(modelName, providerName, messages_, { separateSystemMessage: false })
|
||||
|
||||
const tools = devInfo?.supportsTools && (tools_?.length ?? 0) !== 0 ? tools_?.map(tool => toOpenAITool(tool)) : undefined
|
||||
|
||||
|
||||
const openai: OpenAI = newOpenAI({ providerName, settingsOfProvider })
|
||||
|
|
@ -141,7 +159,9 @@ export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages, on
|
|||
model: modelName,
|
||||
messages: messages,
|
||||
stream: true,
|
||||
tools: tools?.map(tool => toOpenAITool(tool)),
|
||||
tools: tools,
|
||||
tool_choice: tools ? 'auto' : undefined,
|
||||
parallel_tool_calls: tools ? false : undefined,
|
||||
}
|
||||
|
||||
openai.chat.completions
|
||||
|
|
@ -155,9 +175,11 @@ export const sendOpenAIChat: _InternalSendLLMChatMessageFnType = ({ messages, on
|
|||
// tool call
|
||||
for (const tool of chunk.choices[0]?.delta?.tool_calls ?? []) {
|
||||
const index = tool.index
|
||||
if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', args: '' }
|
||||
if (!toolCallOfIndex[index]) toolCallOfIndex[index] = { name: '', args: '', id: '' }
|
||||
toolCallOfIndex[index].name += tool.function?.name ?? ''
|
||||
toolCallOfIndex[index].args += tool.function?.arguments ?? ''
|
||||
toolCallOfIndex[index].args += tool.function?.arguments ?? '';
|
||||
toolCallOfIndex[index].id = tool.id ?? ''
|
||||
|
||||
}
|
||||
|
||||
// message
|
||||
|
|
|
|||
|
|
@ -3,197 +3,12 @@
|
|||
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
|
||||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import { SendLLMMessageParams, OnText, OnFinalMessage, OnError, LLMChatMessage, _InternalLLMChatMessage } from '../../common/llmMessageTypes.js';
|
||||
import { SendLLMMessageParams, OnText, OnFinalMessage, OnError, _InternalLLMChatMessage } from '../../common/llmMessageTypes.js';
|
||||
import { IMetricsService } from '../../common/metricsService.js';
|
||||
import { displayInfoOfProviderName } from '../../common/voidSettingsTypes.js';
|
||||
|
||||
import { sendAnthropicChat } from './anthropic.js';
|
||||
import { sendOllamaFIM, sendOllamaChat } from './ollama.js';
|
||||
import { sendOpenAIChat } from './openai.js';
|
||||
import { sendGeminiChat } from './gemini.js';
|
||||
import { developerInfoOfModelName, developerInfoOfProviderName, displayInfoOfProviderName, ProviderName, recognizedModelOfModelName } from '../../common/voidSettingsTypes.js';
|
||||
|
||||
|
||||
const cleanChatMessages = (modelName: string, providerName: ProviderName, messages: LLMChatMessage[]): { separateSystemMessageStr?: string, messages: _InternalLLMChatMessage[] } => {
|
||||
const recognizedModel = recognizedModelOfModelName(modelName)
|
||||
const { separateSystemMessage, toolsGoInRole, modelOverrides } = developerInfoOfProviderName(providerName)
|
||||
|
||||
const { supportsSystemMessage, maxTokens, /* supportsTools, supportsAutocompleteFIM, supportsStreaming */ } = developerInfoOfModelName(recognizedModel, modelOverrides)
|
||||
|
||||
|
||||
// trim message content (Anthropic and other providers give an error if there is trailing whitespace)
|
||||
messages = messages.map(m => ({ ...m, content: m.content.trim() }))
|
||||
|
||||
|
||||
// 1. SYSTEM MESSAGE
|
||||
// find system messages and concatenate them
|
||||
const systemMessageStr = messages
|
||||
.filter(msg => msg.role === 'system')
|
||||
.map(msg => msg.content)
|
||||
.join('\n') || undefined;
|
||||
|
||||
let separateSystemMessageStr = undefined
|
||||
|
||||
// remove all system messages
|
||||
const noSystemMessages: _InternalLLMChatMessage[] = messages.filter(msg => msg.role !== 'system')
|
||||
|
||||
if (systemMessageStr) {
|
||||
// if supports system message
|
||||
if (supportsSystemMessage) {
|
||||
if (separateSystemMessage)
|
||||
separateSystemMessageStr = systemMessageStr
|
||||
else {
|
||||
noSystemMessages.unshift({ role: supportsSystemMessage, content: systemMessageStr }) // add new first message
|
||||
}
|
||||
}
|
||||
// if does not support system message
|
||||
else {
|
||||
if (supportsSystemMessage) {
|
||||
if (noSystemMessages.length === 0)
|
||||
noSystemMessages.push({ role: 'user', content: systemMessageStr })
|
||||
// add system mesasges to first message (should be a user message)
|
||||
else {
|
||||
const newFirstMessage = {
|
||||
role: noSystemMessages[0].role,
|
||||
content: (''
|
||||
+ '<SYSTEM_MESSAGE>\n'
|
||||
+ systemMessageStr
|
||||
+ '\n'
|
||||
+ '</SYSTEM_MESSAGE>\n'
|
||||
+ noSystemMessages[0].content
|
||||
)
|
||||
}
|
||||
noSystemMessages.splice(0, 1) // delete first message
|
||||
noSystemMessages.unshift(newFirstMessage) // add new first message
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. TOOLS
|
||||
|
||||
const newMessages = noSystemMessages;
|
||||
|
||||
if (toolsGoInRole) {
|
||||
let index = 0;
|
||||
while (index < newMessages.length) {
|
||||
|
||||
// merge tool with the previous assistant and the following user message
|
||||
|
||||
// take prev message and add
|
||||
/*
|
||||
openai assistant message will have: https://platform.openai.com/docs/guides/function-calling#function-calling-steps
|
||||
"tool_calls":[
|
||||
{
|
||||
"id": "call_12345xyz",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": "{\"latitude\":48.8566,\"longitude\":2.3522}"
|
||||
}
|
||||
}]
|
||||
|
||||
openai user response will be:
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tool_call.id,
|
||||
"content": str(result)
|
||||
}
|
||||
|
||||
anthropic assistant message will have: https://docs.anthropic.com/en/docs/build-with-claude/tool-use#tool-use-examples
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "<thinking>I need to call the get_weather function, and the user wants SF, which is likely San Francisco, CA.</thinking>"
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01A09q90qw90lq917835lq9",
|
||||
"name": "get_weather",
|
||||
"input": {"location": "San Francisco, CA", "unit": "celsius"}
|
||||
}
|
||||
]
|
||||
|
||||
anthropic user message response will be:
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01A09q90qw90lq917835lq9",
|
||||
"content": "15 degrees"
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
ACCORDING TO 4o: gemini: similar to openai, but function_call, and only 1 call per message (no id because only 1 message)
|
||||
gemini request: {
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"function_call": {
|
||||
"name": "get_weather",
|
||||
"arguments": {
|
||||
"latitude": 48.8566,
|
||||
"longitude": 2.3522
|
||||
}
|
||||
}
|
||||
}
|
||||
gemini response:
|
||||
{
|
||||
"role": "assistant",
|
||||
"function_response": {
|
||||
"name": "get_weather",
|
||||
"response": {
|
||||
"temperature": "15°C",
|
||||
"condition": "Cloudy"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+ anthropic
|
||||
|
||||
+ openai-compat (4)
|
||||
+ gemini
|
||||
|
||||
ollama
|
||||
|
||||
|
||||
mistral: same as openai
|
||||
|
||||
*/
|
||||
|
||||
|
||||
if (newMessages[index].role === 'tool') {
|
||||
const toolMessage = newMessages[index];
|
||||
const assistantMessage = newMessages[index - 1];
|
||||
const userMessage = newMessages[index + 1];
|
||||
|
||||
// while ((toolIndex = newMessages.findIndex((msg, idx) => idx > toolIndex && msg.role === 'tool')) !== -1) {
|
||||
|
||||
// tool_use goes in assistant
|
||||
if (assistantMessage?.role === 'assistant') {
|
||||
assistantMessage.tool_use += `\n${toolMessage.content}`;
|
||||
}
|
||||
|
||||
// tool_result goes in user
|
||||
if (userMessage?.role === 'user') {
|
||||
|
||||
userMessage.content = `${toolMessage.content}\n${userMessage.content}`;
|
||||
}
|
||||
|
||||
// Remove the tool message after merging its content
|
||||
newMessages.splice(index, 1);
|
||||
} else {
|
||||
index++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
return {
|
||||
separateSystemMessageStr,
|
||||
messages: newMessages
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export const sendLLMMessage = ({
|
||||
|
|
@ -214,16 +29,6 @@ export const sendLLMMessage = ({
|
|||
metricsService: IMetricsService
|
||||
) => {
|
||||
|
||||
let messagesArr: _InternalLLMChatMessage[] = []
|
||||
|
||||
// TODO!!! move this to the actual providers
|
||||
if (messagesType === 'chatMessages') {
|
||||
const { messages: cleanedMessages, separateSystemMessageStr } = cleanChatMessages(modelName, providerName, [
|
||||
{ role: 'system', content: aiInstructions },
|
||||
...messages_
|
||||
])
|
||||
messagesArr = cleanedMessages
|
||||
}
|
||||
|
||||
// only captures number of messages and message "shape", no actual code, instructions, prompts, etc
|
||||
const captureLLMEvent = (eventId: string, extras?: object) => {
|
||||
|
|
@ -231,8 +36,8 @@ export const sendLLMMessage = ({
|
|||
providerName,
|
||||
modelName,
|
||||
...messagesType === 'chatMessages' ? {
|
||||
numMessages: messagesArr?.length,
|
||||
messagesShape: messagesArr?.map(msg => ({ role: msg.role, length: msg.content.length })),
|
||||
numMessages: messages_?.length,
|
||||
messagesShape: messages_?.map(msg => ({ role: msg.role, length: msg.content.length })),
|
||||
origNumMessages: messages_?.length,
|
||||
origMessagesShape: messages_?.map(msg => ({ role: msg.role, length: msg.content.length })),
|
||||
|
||||
|
|
@ -283,7 +88,10 @@ export const sendLLMMessage = ({
|
|||
}
|
||||
abortRef_.current = onAbort
|
||||
|
||||
captureLLMEvent(`${loggingName} - Sending Message`, { messageLength: messagesArr[messagesArr.length - 1]?.content.length })
|
||||
if (messagesType === 'chatMessages')
|
||||
captureLLMEvent(`${loggingName} - Sending Message`, { messageLength: messages_[messages_.length - 1]?.content.length })
|
||||
else if (messagesType === 'FIMMessage')
|
||||
captureLLMEvent(`${loggingName} - Sending FIM`, {}) // TODO!!! add more metrics
|
||||
|
||||
try {
|
||||
switch (providerName) {
|
||||
|
|
@ -292,21 +100,15 @@ export const sendLLMMessage = ({
|
|||
case 'deepseek':
|
||||
case 'openAICompatible':
|
||||
case 'mistral':
|
||||
case 'groq':
|
||||
if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI FIM', tools: [] })
|
||||
else /* */ sendOpenAIChat({ messages: messagesArr, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools });
|
||||
break;
|
||||
case 'ollama':
|
||||
if (messagesType === 'FIMMessage') sendOllamaFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName });
|
||||
else /* */ sendOllamaChat({ messages: messagesArr, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools });
|
||||
case 'groq':
|
||||
case 'gemini':
|
||||
if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - OpenAI FIM', tools: [] })
|
||||
else /* */ sendOpenAIChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools });
|
||||
break;
|
||||
case 'anthropic':
|
||||
if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Anthropic FIM', tools: [] })
|
||||
else /* */ sendAnthropicChat({ messages: messagesArr, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools });
|
||||
break;
|
||||
case 'gemini':
|
||||
if (messagesType === 'FIMMessage') onFinalMessage({ fullText: 'TODO - Gemini FIM', tools: [] })
|
||||
else /* */ sendGeminiChat({ messages: messagesArr, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools });
|
||||
else /* */ sendAnthropicChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName, _setAborter, providerName, tools });
|
||||
break;
|
||||
default:
|
||||
onError({ message: `Error: Void provider was "${providerName}", which is not recognized.`, fullError: null })
|
||||
|
|
|
|||
Loading…
Reference in a new issue