mirror of
https://github.com/voideditor/void
synced 2026-05-23 01:18:25 +00:00
add GCP vertex (w/ auth), MSFT azure, litellm, lm studio - NEEDS TESTING
This commit is contained in:
Andrew Pareles
parent
7d3ce48f5f
commit
6d28b0627f
14 changed files with 463 additions and 110 deletions
159
package-lock.json
generated
159
package-lock.json
generated
|
|
@ -45,6 +45,7 @@
|
|||
"cross-spawn": "^7.0.6",
|
||||
"diff": "^7.0.0",
|
||||
"eslint-plugin-react": "^7.37.4",
|
||||
"google-auth-library": "^9.15.1",
|
||||
"groq-sdk": "^0.15.0",
|
||||
"http-proxy-agent": "^7.0.0",
|
||||
"https-proxy-agent": "^7.0.2",
|
||||
|
|
@ -5991,6 +5992,15 @@
|
|||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/bignumber.js": {
|
||||
"version": "9.2.1",
|
||||
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.2.1.tgz",
|
||||
"integrity": "sha512-+NzaKgOUvInq9TIUZ1+DRspzf/HApkCwD4btfuasFTdrfnOxqx853TgDpMolp+uv4RpRp7bPcEU2zKr9+fRmyw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/binary-extensions": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
|
||||
|
|
@ -6221,6 +6231,12 @@
|
|||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/buffer-equal-constant-time": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
|
||||
"integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/buffer-from": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
|
||||
|
|
@ -8068,6 +8084,15 @@
|
|||
"integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/ecdsa-sig-formatter": {
|
||||
"version": "1.0.11",
|
||||
"resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
|
||||
"integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/editorconfig": {
|
||||
"version": "0.15.2",
|
||||
"resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.2.tgz",
|
||||
|
|
@ -9284,8 +9309,7 @@
|
|||
"node_modules/extend": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
|
||||
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
|
||||
"dev": true
|
||||
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
|
||||
},
|
||||
"node_modules/extend-shallow": {
|
||||
"version": "3.0.2",
|
||||
|
|
@ -10205,6 +10229,68 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/gaxios": {
|
||||
"version": "6.7.1",
|
||||
"resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz",
|
||||
"integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"extend": "^3.0.2",
|
||||
"https-proxy-agent": "^7.0.1",
|
||||
"is-stream": "^2.0.0",
|
||||
"node-fetch": "^2.6.9",
|
||||
"uuid": "^9.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/gaxios/node_modules/is-stream": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
|
||||
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/gaxios/node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/gcp-metadata": {
|
||||
"version": "6.1.1",
|
||||
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz",
|
||||
"integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"gaxios": "^6.1.1",
|
||||
"google-logging-utils": "^0.0.2",
|
||||
"json-bigint": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/gensync": {
|
||||
"version": "1.0.0-beta.2",
|
||||
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
|
||||
|
|
@ -10841,6 +10927,32 @@
|
|||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/google-auth-library": {
|
||||
"version": "9.15.1",
|
||||
"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz",
|
||||
"integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"base64-js": "^1.3.0",
|
||||
"ecdsa-sig-formatter": "^1.0.11",
|
||||
"gaxios": "^6.1.1",
|
||||
"gcp-metadata": "^6.1.0",
|
||||
"gtoken": "^7.0.0",
|
||||
"jws": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/google-logging-utils": {
|
||||
"version": "0.0.2",
|
||||
"resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz",
|
||||
"integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/gopd": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
|
||||
|
|
@ -10913,6 +11025,19 @@
|
|||
"undici-types": "~5.26.4"
|
||||
}
|
||||
},
|
||||
"node_modules/gtoken": {
|
||||
"version": "7.1.0",
|
||||
"resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz",
|
||||
"integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"gaxios": "^6.0.0",
|
||||
"jws": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/gulp": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/gulp/-/gulp-4.0.2.tgz",
|
||||
|
|
@ -13873,6 +13998,15 @@
|
|||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/json-bigint": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz",
|
||||
"integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"bignumber.js": "^9.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/json-buffer": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
|
||||
|
|
@ -13993,6 +14127,27 @@
|
|||
"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/jwa": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
|
||||
"integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-equal-constant-time": "1.0.1",
|
||||
"ecdsa-sig-formatter": "1.0.11",
|
||||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/jws": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz",
|
||||
"integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"jwa": "^2.0.0",
|
||||
"safe-buffer": "^5.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/kerberos": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/kerberos/-/kerberos-2.1.1.tgz",
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@
|
|||
"cross-spawn": "^7.0.6",
|
||||
"diff": "^7.0.0",
|
||||
"eslint-plugin-react": "^7.37.4",
|
||||
"google-auth-library": "^9.15.1",
|
||||
"groq-sdk": "^0.15.0",
|
||||
"http-proxy-agent": "^7.0.0",
|
||||
"https-proxy-agent": "^7.0.2",
|
||||
|
|
|
|||
|
|
@ -709,7 +709,7 @@ export const VoidCustomDropdownBox = <T extends NonNullable<any>>({
|
|||
</svg>
|
||||
)}
|
||||
</div>
|
||||
<span className="flex justify-between w-full">
|
||||
<span className="flex justify-between items-center w-full gap-x-1">
|
||||
<span>{optionName}</span>
|
||||
<span className='text-void-fg-4 opacity-60'>{optionDetail}</span>
|
||||
</span>
|
||||
|
|
|
|||
|
|
@ -497,7 +497,7 @@ const VoidOnboardingContent = () => {
|
|||
|
||||
const providerNamesOfWantToUseOption: { [wantToUseOption in WantToUseOption]: ProviderName[] } = {
|
||||
smart: ['anthropic', 'openAI', 'gemini', 'openRouter'],
|
||||
private: ['ollama', 'vLLM', 'openAICompatible'],
|
||||
private: ['ollama', 'vLLM', 'openAICompatible', 'lmStudio'],
|
||||
cheap: ['gemini', 'deepseek', 'openRouter', 'ollama', 'vLLM'],
|
||||
all: providerNames,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
*--------------------------------------------------------------------------------------*/
|
||||
|
||||
import React, { useCallback, useEffect, useMemo, useState } from 'react'
|
||||
import { ProviderName, SettingName, displayInfoOfSettingName, providerNames, VoidStatefulModelInfo, customSettingNamesOfProvider, RefreshableProviderName, refreshableProviderNames, displayInfoOfProviderName, nonlocalProviderNames, localProviderNames, GlobalSettingName, featureNames, displayInfoOfFeatureName, isProviderNameDisabled, FeatureName, hasDownloadButtonsOnModelsProviderNames } from '../../../../common/voidSettingsTypes.js'
|
||||
import { ProviderName, SettingName, displayInfoOfSettingName, providerNames, VoidStatefulModelInfo, customSettingNamesOfProvider, RefreshableProviderName, refreshableProviderNames, displayInfoOfProviderName, nonlocalProviderNames, localProviderNames, GlobalSettingName, featureNames, displayInfoOfFeatureName, isProviderNameDisabled, FeatureName, hasDownloadButtonsOnModelsProviderNames, subTextMdOfProviderName } from '../../../../common/voidSettingsTypes.js'
|
||||
import ErrorBoundary from '../sidebar-tsx/ErrorBoundary.js'
|
||||
import { VoidButtonBgDarken, VoidCustomDropdownBox, VoidInputBox2, VoidSimpleInputBox, VoidSwitch } from '../util/inputs.js'
|
||||
import { useAccessor, useIsDark, useRefreshModelListener, useRefreshModelState, useSettingsState } from '../util/services.js'
|
||||
|
|
@ -344,9 +344,9 @@ export const ModelDump = () => {
|
|||
|
||||
// providers
|
||||
|
||||
const ProviderSetting = ({ providerName, settingName }: { providerName: ProviderName, settingName: SettingName }) => {
|
||||
const ProviderSetting = ({ providerName, settingName, subTextMd }: { providerName: ProviderName, settingName: SettingName, subTextMd: React.ReactNode }) => {
|
||||
|
||||
const { title: settingTitle, placeholder, isPasswordField, subTextMd } = displayInfoOfSettingName(providerName, settingName)
|
||||
const { title: settingTitle, placeholder, isPasswordField } = displayInfoOfSettingName(providerName, settingName)
|
||||
|
||||
const accessor = useAccessor()
|
||||
const voidSettingsService = accessor.get('IVoidSettingsService')
|
||||
|
|
@ -370,10 +370,9 @@ const ProviderSetting = ({ providerName, settingName }: { providerName: Provider
|
|||
passwordBlur={isPasswordField}
|
||||
compact={true}
|
||||
/>
|
||||
{subTextMd === undefined ? null : <div className='py-1 px-3 opacity-50 text-sm'>
|
||||
<ChatMarkdownRender string={subTextMd} chatMessageLocation={undefined} />
|
||||
{!subTextMd ? null : <div className='py-1 px-3 opacity-50 text-sm'>
|
||||
{subTextMd}
|
||||
</div>}
|
||||
|
||||
</div>
|
||||
</ErrorBoundary>
|
||||
}
|
||||
|
|
@ -456,7 +455,14 @@ export const SettingsForProvider = ({ providerName, showProviderTitle, showProvi
|
|||
<div className='px-0'>
|
||||
{/* settings besides models (e.g. api key) */}
|
||||
{settingNames.map((settingName, i) => {
|
||||
return <ProviderSetting key={settingName} providerName={providerName} settingName={settingName} />
|
||||
|
||||
return <ProviderSetting
|
||||
key={settingName}
|
||||
providerName={providerName}
|
||||
settingName={settingName}
|
||||
subTextMd={i !== settingNames.length - 1 ? null
|
||||
: <ChatMarkdownRender string={subTextMdOfProviderName(providerName)} chatMessageLocation={undefined} />}
|
||||
/>
|
||||
})}
|
||||
|
||||
{showProviderSuggestions && needsModel ?
|
||||
|
|
|
|||
|
|
@ -43,7 +43,22 @@ export const defaultProviderSettings = {
|
|||
},
|
||||
mistral: {
|
||||
apiKey: '',
|
||||
}
|
||||
},
|
||||
lmStudio: {
|
||||
endpoint: 'http://localhost:1234',
|
||||
},
|
||||
liteLLM: { // https://docs.litellm.ai/docs/providers/openai_compatible
|
||||
endpoint: '',
|
||||
},
|
||||
googleVertex: { // google https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library
|
||||
region: 'us-west2',
|
||||
project: '',
|
||||
},
|
||||
microsoftAzure: { // microsoft Azure Foundry
|
||||
project: '', // really 'resource'
|
||||
apiKey: '',
|
||||
azureApiVersion: '2024-05-01-preview',
|
||||
},
|
||||
} as const
|
||||
|
||||
|
||||
|
|
@ -84,6 +99,8 @@ export const defaultModelsOfProvider = {
|
|||
],
|
||||
vLLM: [ // autodetected
|
||||
],
|
||||
lmStudio: [], // autodetected
|
||||
|
||||
openRouter: [ // https://openrouter.ai/models
|
||||
// 'anthropic/claude-3.7-sonnet:thinking',
|
||||
'anthropic/claude-3.7-sonnet',
|
||||
|
|
@ -112,6 +129,11 @@ export const defaultModelsOfProvider = {
|
|||
'ministral-8b-latest',
|
||||
],
|
||||
openAICompatible: [], // fallback
|
||||
googleVertex: [],
|
||||
microsoftAzure: [],
|
||||
liteLLM: [],
|
||||
|
||||
|
||||
} as const satisfies Record<ProviderName, string[]>
|
||||
|
||||
|
||||
|
|
@ -618,6 +640,7 @@ const geminiModelOptions = { // https://ai.google.dev/gemini-api/docs/pricing
|
|||
downloadable: false,
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
specialToolFormat: 'openai-style',
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'gemini-2.0-flash': {
|
||||
|
|
@ -806,6 +829,25 @@ const groqSettings: VoidStaticProviderInfo = {
|
|||
modelOptionsFallback: (modelName) => { return null }
|
||||
}
|
||||
|
||||
|
||||
// ---------------- GOOGLE VERTEX ----------------
|
||||
const googleVertexModelOptions = {
|
||||
} as const satisfies Record<string, VoidStaticModelInfo>
|
||||
const googleVertexSettings: VoidStaticProviderInfo = {
|
||||
modelOptions: googleVertexModelOptions,
|
||||
modelOptionsFallback: (modelName) => { return null }
|
||||
}
|
||||
|
||||
// ---------------- MICROSOFT AZURE ----------------
|
||||
const microsoftAzureModelOptions = {
|
||||
} as const satisfies Record<string, VoidStaticModelInfo>
|
||||
const microsoftAzureSettings: VoidStaticProviderInfo = {
|
||||
modelOptions: microsoftAzureModelOptions,
|
||||
modelOptionsFallback: (modelName) => { return null }
|
||||
}
|
||||
|
||||
|
||||
// ---------------- VLLM, OLLAMA, OPENAICOMPAT (self-hosted / local) ----------------
|
||||
const ollamaModelOptions = {
|
||||
'qwen2.5-coder:1.5b': {
|
||||
contextWindow: 32_000,
|
||||
|
|
@ -858,9 +900,6 @@ const ollamaModelOptions = {
|
|||
export const ollamaRecommendedModels = ['qwen2.5-coder:1.5b', 'llama3.1', 'qwq', 'deepseek-r1'] as const satisfies (keyof typeof ollamaModelOptions)[]
|
||||
|
||||
|
||||
|
||||
// ---------------- VLLM, OLLAMA, OPENAICOMPAT (self-hosted / local) ----------------
|
||||
|
||||
const vLLMSettings: VoidStaticProviderInfo = {
|
||||
// reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#streaming-chat-completions
|
||||
providerReasoningIOSettings: { output: { nameOfFieldInDelta: 'reasoning_content' }, },
|
||||
|
|
@ -868,6 +907,18 @@ const vLLMSettings: VoidStaticProviderInfo = {
|
|||
modelOptions: {}, // TODO
|
||||
}
|
||||
|
||||
const lmStudioSettings: VoidStaticProviderInfo = {
|
||||
providerReasoningIOSettings: { output: { nameOfFieldInDelta: 'reasoning_content' }, },
|
||||
modelOptionsFallback: (modelName) => extensiveModelFallback(modelName, { downloadable: { sizeGb: 'not-known' } }),
|
||||
modelOptions: {}, // TODO
|
||||
}
|
||||
|
||||
const liteLLMSettings: VoidStaticProviderInfo = {
|
||||
providerReasoningIOSettings: { output: { nameOfFieldInDelta: 'reasoning_content' }, },
|
||||
modelOptionsFallback: (modelName) => extensiveModelFallback(modelName, { downloadable: { sizeGb: 'not-known' } }),
|
||||
modelOptions: {}, // TODO
|
||||
}
|
||||
|
||||
const ollamaSettings: VoidStaticProviderInfo = {
|
||||
// reasoning: we need to filter out reasoning <think> tags manually
|
||||
providerReasoningIOSettings: { output: { needsManualParse: true }, },
|
||||
|
|
@ -1027,9 +1078,12 @@ const modelSettingsOfProvider: { [providerName in ProviderName]: VoidStaticProvi
|
|||
ollama: ollamaSettings,
|
||||
openAICompatible: openaiCompatible,
|
||||
mistral: mistralSettings,
|
||||
// googleVertex: {},
|
||||
// microsoftAzure: {},
|
||||
// openHands: {},
|
||||
|
||||
liteLLM: liteLLMSettings,
|
||||
lmStudio: lmStudioSettings,
|
||||
|
||||
googleVertex: googleVertexSettings,
|
||||
microsoftAzure: microsoftAzureSettings,
|
||||
} as const
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -330,6 +330,7 @@ Here's an example of a good edit suggestion:
|
|||
${fileNameEditExample}.`)
|
||||
}
|
||||
|
||||
details.push(`NEVER write the FULL PATH of a file when speaking with the user. Just write the file name ONLY.`)
|
||||
details.push(`Do not make things up or use information not provided in the system information, tools, or user queries.`)
|
||||
details.push(`Today's date is ${new Date().toDateString()}.`)
|
||||
|
||||
|
|
@ -459,9 +460,9 @@ ${tripleTick[1]}
|
|||
|
||||
1. The change to make will be labeled \`CHANGE\` and the original file will be labeled \`ORIGINAL_FILE\`.
|
||||
|
||||
2. You are allowed to output multiple SEARCH/REPLACE blocks.
|
||||
2. Your SEARCH/REPLACE block(s) must implement the change EXACTLY. Do not introduce (or omit) any new comments, spaces, or whitespace.
|
||||
|
||||
3. Your SEARCH/REPLACE block(s) must implement the change EXACTLY. Do not introduce (or omit) any new comments, spaces, or whitespace.
|
||||
3. You are allowed to output multiple SEARCH/REPLACE blocks.
|
||||
|
||||
4. Your output should consist ONLY of SEARCH/REPLACE blocks. Do NOT output any text or explanations before or after this.
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import { ILLMMessageService } from './sendLLMMessageService.js';
|
|||
import { Emitter, Event } from '../../../../base/common/event.js';
|
||||
import { Disposable, IDisposable } from '../../../../base/common/lifecycle.js';
|
||||
import { RefreshableProviderName, refreshableProviderNames, SettingsOfProvider } from './voidSettingsTypes.js';
|
||||
import { OllamaModelResponse, VLLMModelResponse } from './sendLLMMessageTypes.js';
|
||||
import { LMStudioModelResponse, OllamaModelResponse, VLLMModelResponse } from './sendLLMMessageTypes.js';
|
||||
import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js';
|
||||
import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
|
||||
|
||||
|
|
@ -46,6 +46,7 @@ export type RefreshModelStateOfProvider = Record<RefreshableProviderName, Refres
|
|||
const refreshBasedOn: { [k in RefreshableProviderName]: (keyof SettingsOfProvider[k])[] } = {
|
||||
ollama: ['_didFillInProviderSettings', 'endpoint'],
|
||||
vLLM: ['_didFillInProviderSettings', 'endpoint'],
|
||||
lmStudio: ['_didFillInProviderSettings', 'endpoint'],
|
||||
// openAICompatible: ['_didFillInProviderSettings', 'endpoint', 'apiKey'],
|
||||
}
|
||||
const REFRESH_INTERVAL = 5_000
|
||||
|
|
@ -142,6 +143,7 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ
|
|||
state: RefreshModelStateOfProvider = {
|
||||
ollama: { state: 'init', timeoutId: null },
|
||||
vLLM: { state: 'init', timeoutId: null },
|
||||
lmStudio: { state: 'init', timeoutId: null },
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -172,6 +174,7 @@ export class RefreshModelService extends Disposable implements IRefreshModelServ
|
|||
models.map(model => {
|
||||
if (providerName === 'ollama') return (model as OllamaModelResponse).name;
|
||||
else if (providerName === 'vLLM') return (model as VLLMModelResponse).id;
|
||||
else if (providerName === 'lmStudio') return (model as LMStudioModelResponse).id;
|
||||
else throw new Error('refreshMode fn: unknown provider', providerName);
|
||||
}),
|
||||
{ enableProviderOnSuccess: options.enableProviderOnSuccess, hideRefresh: options.doNotFire }
|
||||
|
|
|
|||
|
|
@ -170,6 +170,7 @@ type OpenaiCompatibleModelResponse = {
|
|||
}
|
||||
|
||||
export type VLLMModelResponse = OpenaiCompatibleModelResponse
|
||||
export type LMStudioModelResponse = OpenaiCompatibleModelResponse
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -245,17 +245,36 @@ class VoidSettingsService extends Disposable implements IVoidSettingsService {
|
|||
}
|
||||
|
||||
// the stored data structure might be outdated, so we need to update it here
|
||||
readS = {
|
||||
...readS,
|
||||
settingsOfProvider: {
|
||||
try {
|
||||
readS = {
|
||||
...readS,
|
||||
...defaultSettingsOfProvider,
|
||||
...readS.settingsOfProvider,
|
||||
mistral: { // we added mistral
|
||||
...defaultSettingsOfProvider.mistral,
|
||||
...readS.settingsOfProvider.mistral,
|
||||
},
|
||||
} // we added mistral
|
||||
}
|
||||
|
||||
for (const providerName of providerNames) {
|
||||
readS.settingsOfProvider[providerName] = {
|
||||
...defaultSettingsOfProvider[providerName],
|
||||
...readS.settingsOfProvider[providerName],
|
||||
} as any
|
||||
}
|
||||
// readS = {
|
||||
// ...readS,
|
||||
// settingsOfProvider: {
|
||||
// ...defaultSettingsOfProvider,
|
||||
// ...readS.settingsOfProvider,
|
||||
// mistral: { // we added mistral
|
||||
// ...defaultSettingsOfProvider.mistral,
|
||||
// ...readS.settingsOfProvider.mistral,
|
||||
// },
|
||||
// } // we added mistral
|
||||
// }
|
||||
}
|
||||
|
||||
catch (e) {
|
||||
readS = defaultState()
|
||||
}
|
||||
|
||||
this.state = readS
|
||||
this.state = _stateWithUpdatedDefaultModels(this.state)
|
||||
this.state = _validatedModelState(this.state);
|
||||
|
|
@ -263,6 +282,7 @@ class VoidSettingsService extends Disposable implements IVoidSettingsService {
|
|||
|
||||
this._resolver();
|
||||
this._onDidChangeState.fire();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ type UnionOfKeys<T> = T extends T ? keyof T : never;
|
|||
export type ProviderName = keyof typeof defaultProviderSettings
|
||||
export const providerNames = Object.keys(defaultProviderSettings) as ProviderName[]
|
||||
|
||||
export const localProviderNames = ['ollama', 'vLLM'] satisfies ProviderName[] // all local names
|
||||
export const localProviderNames = ['ollama', 'vLLM', 'lmStudio'] satisfies ProviderName[] // all local names
|
||||
export const nonlocalProviderNames = providerNames.filter((name) => !(localProviderNames as string[]).includes(name)) // all non-local names
|
||||
|
||||
type CustomSettingName = UnionOfKeys<typeof defaultProviderSettings[ProviderName]>
|
||||
|
|
@ -59,74 +59,78 @@ type DisplayInfoForProviderName = {
|
|||
|
||||
export const displayInfoOfProviderName = (providerName: ProviderName): DisplayInfoForProviderName => {
|
||||
if (providerName === 'anthropic') {
|
||||
return {
|
||||
title: 'Anthropic',
|
||||
}
|
||||
return { title: 'Anthropic', }
|
||||
}
|
||||
else if (providerName === 'openAI') {
|
||||
return {
|
||||
title: 'OpenAI',
|
||||
}
|
||||
return { title: 'OpenAI', }
|
||||
}
|
||||
else if (providerName === 'deepseek') {
|
||||
return {
|
||||
// title: 'DeepSeek.com API',
|
||||
title: 'DeepSeek',
|
||||
}
|
||||
return { title: 'DeepSeek', }
|
||||
}
|
||||
else if (providerName === 'openRouter') {
|
||||
return {
|
||||
title: 'OpenRouter',
|
||||
}
|
||||
return { title: 'OpenRouter', }
|
||||
}
|
||||
else if (providerName === 'ollama') {
|
||||
return {
|
||||
title: 'Ollama',
|
||||
}
|
||||
return { title: 'Ollama', }
|
||||
}
|
||||
else if (providerName === 'vLLM') {
|
||||
return {
|
||||
title: 'vLLM',
|
||||
}
|
||||
return { title: 'vLLM', }
|
||||
}
|
||||
else if (providerName === 'liteLLM') {
|
||||
return { title: 'LiteLLM', }
|
||||
}
|
||||
else if (providerName === 'lmStudio') {
|
||||
return { title: 'LM Studio', }
|
||||
}
|
||||
else if (providerName === 'openAICompatible') {
|
||||
return {
|
||||
title: 'OpenAI-Compatible',
|
||||
}
|
||||
return { title: 'OpenAI-Compatible', }
|
||||
}
|
||||
else if (providerName === 'gemini') {
|
||||
return {
|
||||
// title: 'Gemini API',
|
||||
title: 'Gemini',
|
||||
}
|
||||
return { title: 'Gemini', }
|
||||
}
|
||||
else if (providerName === 'groq') {
|
||||
return {
|
||||
// title: 'Groq.com API',
|
||||
title: 'Groq',
|
||||
}
|
||||
return { title: 'Groq', }
|
||||
}
|
||||
else if (providerName === 'xAI') {
|
||||
return {
|
||||
// title: 'Grok (xAI)',
|
||||
title: 'xAI',
|
||||
}
|
||||
return { title: 'xAI', }
|
||||
}
|
||||
else if (providerName === 'mistral') {
|
||||
return {
|
||||
// title: 'Mistral API',
|
||||
title: 'Mistral',
|
||||
}
|
||||
return { title: 'Mistral', }
|
||||
}
|
||||
else if (providerName === 'googleVertex') {
|
||||
return { title: 'Google Vertex AI', }
|
||||
}
|
||||
else if (providerName === 'microsoftAzure') {
|
||||
return { title: 'Microsoft Azure OpenAI', }
|
||||
}
|
||||
|
||||
|
||||
throw new Error(`descOfProviderName: Unknown provider name: "${providerName}"`)
|
||||
}
|
||||
|
||||
export const subTextMdOfProviderName = (providerName: ProviderName): string => {
|
||||
|
||||
if (providerName === 'anthropic') return 'Get your [API Key here](https://console.anthropic.com/settings/keys).'
|
||||
if (providerName === 'openAI') return 'Get your [API Key here](https://platform.openai.com/api-keys).'
|
||||
if (providerName === 'deepseek') return 'Get your [API Key here](https://platform.deepseek.com/api_keys).'
|
||||
if (providerName === 'openRouter') return 'Get your [API Key here](https://openrouter.ai/settings/keys).'
|
||||
if (providerName === 'gemini') return 'Get your [API Key here](https://aistudio.google.com/apikey).'
|
||||
if (providerName === 'groq') return 'Get your [API Key here](https://console.groq.com/keys).'
|
||||
if (providerName === 'xAI') return 'Get your [API Key here](https://console.x.ai).'
|
||||
if (providerName === 'mistral') return 'Get your [API Key here](https://console.mistral.ai/api-keys).'
|
||||
if (providerName === 'openAICompatible') return `Use any OpenAI-compatible endpoint (LM Studio, LiteLM, etc).`
|
||||
if (providerName === 'googleVertex') return 'You must authenticate before using Vertex with Void. Read more about endpoints [here](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), and regions [here](https://cloud.google.com/vertex-ai/docs/general/locations#available-regions).'
|
||||
if (providerName === 'microsoftAzure') return 'Read more about endpoints [here](https://learn.microsoft.com/en-us/rest/api/aifoundry/model-inference/get-chat-completions/get-chat-completions?view=rest-aifoundry-model-inference-2024-05-01-preview&tabs=HTTP), and get your API key [here](https://learn.microsoft.com/en-us/azure/search/search-security-api-keys?tabs=rest-use%2Cportal-find%2Cportal-query#find-existing-keys).'
|
||||
if (providerName === 'ollama') return 'If you would like to change this endpoint, please read more about [Endpoints here](https://github.com/ollama/ollama/blob/main/docs/faq.md#how-can-i-expose-ollama-on-my-network).'
|
||||
if (providerName === 'vLLM') return 'If you would like to change this endpoint, please read more about [Endpoints here](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server).'
|
||||
if (providerName === 'lmStudio') return 'If you would like to change this endpoint, please more about [Endpoints here](https://lmstudio.ai/docs/app/api/endpoints/openai).'
|
||||
if (providerName === 'liteLLM') return 'Read more about endpoints [here](https://docs.litellm.ai/docs/providers/openai_compatible).'
|
||||
|
||||
throw new Error(`subTextMdOfProviderName: Unknown provider name: "${providerName}"`)
|
||||
}
|
||||
|
||||
type DisplayInfo = {
|
||||
title: string;
|
||||
placeholder: string;
|
||||
subTextMd?: string;
|
||||
isPasswordField?: boolean;
|
||||
}
|
||||
export const displayInfoOfSettingName = (providerName: ProviderName, settingName: SettingName): DisplayInfo => {
|
||||
|
|
@ -140,23 +144,14 @@ export const displayInfoOfSettingName = (providerName: ProviderName, settingName
|
|||
providerName === 'openAI' ? 'sk-proj-key...' :
|
||||
providerName === 'deepseek' ? 'sk-key...' :
|
||||
providerName === 'openRouter' ? 'sk-or-key...' : // sk-or-v1-key
|
||||
providerName === 'gemini' ? 'key...' :
|
||||
providerName === 'gemini' ? 'AIzaSy...' :
|
||||
providerName === 'groq' ? 'gsk_key...' :
|
||||
providerName === 'openAICompatible' ? 'sk-key...' :
|
||||
providerName === 'xAI' ? 'xai-key...' :
|
||||
providerName === 'mistral' ? 'api-key...' :
|
||||
'',
|
||||
providerName === 'googleVertex' ? 'AIzaSy...' :
|
||||
'',
|
||||
|
||||
subTextMd: providerName === 'anthropic' ? 'Get your [API Key here](https://console.anthropic.com/settings/keys).' :
|
||||
providerName === 'openAI' ? 'Get your [API Key here](https://platform.openai.com/api-keys).' :
|
||||
providerName === 'deepseek' ? 'Get your [API Key here](https://platform.deepseek.com/api_keys).' :
|
||||
providerName === 'openRouter' ? 'Get your [API Key here](https://openrouter.ai/settings/keys).' :
|
||||
providerName === 'gemini' ? 'Get your [API Key here](https://aistudio.google.com/apikey).' :
|
||||
providerName === 'groq' ? 'Get your [API Key here](https://console.groq.com/keys).' :
|
||||
providerName === 'xAI' ? 'Get your [API Key here](https://console.x.ai).' :
|
||||
providerName === 'mistral' ? 'Get your [API Key here](https://console.mistral.ai/api-keys).' :
|
||||
providerName === 'openAICompatible' ? `Use any OpenAI-compatible endpoint (LM Studio, LiteLM, etc).` :
|
||||
'',
|
||||
isPasswordField: true,
|
||||
}
|
||||
}
|
||||
|
|
@ -164,19 +159,51 @@ export const displayInfoOfSettingName = (providerName: ProviderName, settingName
|
|||
return {
|
||||
title: providerName === 'ollama' ? 'Endpoint' :
|
||||
providerName === 'vLLM' ? 'Endpoint' :
|
||||
providerName === 'openAICompatible' ? 'baseURL' : // (do not include /chat/completions)
|
||||
'(never)',
|
||||
providerName === 'lmStudio' ? 'Endpoint' :
|
||||
providerName === 'openAICompatible' ? 'baseURL' : // (do not include /chat/completions)
|
||||
providerName === 'googleVertex' ? 'baseURL' :
|
||||
providerName === 'microsoftAzure' ? 'baseURL' :
|
||||
providerName === 'liteLLM' ? 'baseURL' :
|
||||
'(never)',
|
||||
|
||||
placeholder: providerName === 'ollama' ? defaultProviderSettings.ollama.endpoint
|
||||
: providerName === 'vLLM' ? defaultProviderSettings.vLLM.endpoint
|
||||
: providerName === 'openAICompatible' ? 'https://my-website.com/v1'
|
||||
: '(never)',
|
||||
: providerName === 'lmStudio' ? defaultProviderSettings.lmStudio.endpoint
|
||||
: providerName === 'liteLLM' ? 'http://localhost:4000'
|
||||
: '(never)',
|
||||
|
||||
|
||||
subTextMd: providerName === 'ollama' ? 'If you would like to change this endpoint, please read more about [Endpoints here](https://github.com/ollama/ollama/blob/main/docs/faq.md#how-can-i-expose-ollama-on-my-network).' :
|
||||
providerName === 'vLLM' ? 'If you would like to change this endpoint, please read more about [Endpoints here](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server).' :
|
||||
undefined,
|
||||
}
|
||||
}
|
||||
else if (settingName === 'region') {
|
||||
// vertex only
|
||||
return {
|
||||
title: 'Region',
|
||||
placeholder: providerName === 'googleVertex' ? defaultProviderSettings.googleVertex.region
|
||||
: ''
|
||||
}
|
||||
}
|
||||
else if (settingName === 'azureApiVersion') {
|
||||
// azure only
|
||||
return {
|
||||
title: 'API Version',
|
||||
placeholder: providerName === 'microsoftAzure' ? defaultProviderSettings.microsoftAzure.azureApiVersion
|
||||
: ''
|
||||
}
|
||||
}
|
||||
else if (settingName === 'project') {
|
||||
return {
|
||||
title: providerName === 'googleVertex' ? 'Project'
|
||||
: providerName === 'microsoftAzure' ? 'Resource'
|
||||
: '',
|
||||
placeholder: providerName === 'googleVertex' ? 'my-project'
|
||||
: providerName === 'microsoftAzure' ? 'my-resource'
|
||||
: ''
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
else if (settingName === '_didFillInProviderSettings') {
|
||||
return {
|
||||
title: '(never)',
|
||||
|
|
@ -200,6 +227,9 @@ export const displayInfoOfSettingName = (providerName: ProviderName, settingName
|
|||
const defaultCustomSettings: Record<CustomSettingName, undefined> = {
|
||||
apiKey: undefined,
|
||||
endpoint: undefined,
|
||||
region: undefined,
|
||||
project: undefined,
|
||||
azureApiVersion: undefined,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -252,6 +282,18 @@ export const defaultSettingsOfProvider: SettingsOfProvider = {
|
|||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.mistral),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
liteLLM: {
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.liteLLM,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.liteLLM),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
lmStudio: {
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.lmStudio,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.lmStudio),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
groq: { // aggregator (serves models from multiple providers)
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.groq,
|
||||
|
|
@ -282,6 +324,18 @@ export const defaultSettingsOfProvider: SettingsOfProvider = {
|
|||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.vLLM),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
googleVertex: { // aggregator (serves models from multiple providers)
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.googleVertex,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.googleVertex),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
microsoftAzure: { // aggregator (serves models from multiple providers)
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.microsoftAzure,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.microsoftAzure),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { Ollama } from 'ollama';
|
|||
import OpenAI, { ClientOptions } from 'openai';
|
||||
import { MistralCore } from '@mistralai/mistralai/core.js';
|
||||
import { fimComplete } from '@mistralai/mistralai/funcs/fimComplete.js';
|
||||
import { GoogleAuth } from 'google-auth-library'
|
||||
/* eslint-enable */
|
||||
|
||||
import { AnthropicLLMChatMessage, LLMChatMessage, LLMFIMMessage, ModelListParams, OllamaModelResponse, OnError, OnFinalMessage, OnText, RawToolCallObj, RawToolParamsObj } from '../../common/sendLLMMessageTypes.js';
|
||||
|
|
@ -19,6 +20,8 @@ import { extractReasoningWrapper, extractXMLToolsWrapper } from './extractGramma
|
|||
import { availableTools, InternalToolInfo, isAToolName, ToolParamName, voidTools } from '../../common/prompt/prompts.js';
|
||||
|
||||
|
||||
|
||||
|
||||
type InternalCommonMessageParams = {
|
||||
onText: OnText;
|
||||
onFinalMessage: OnFinalMessage;
|
||||
|
|
@ -39,7 +42,16 @@ const invalidApiKeyMessage = (providerName: ProviderName) => `Invalid ${displayI
|
|||
|
||||
// ------------ OPENAI-COMPATIBLE (HELPERS) ------------
|
||||
|
||||
const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPayload }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName, includeInPayload?: { [s: string]: any } }) => {
|
||||
const getGoogleApiKey = async () => {
|
||||
// module‑level singleton
|
||||
const auth = new GoogleAuth({ scopes: `https://www.googleapis.com/auth/cloud-platform` });
|
||||
const key = await auth.getAccessToken()
|
||||
if (!key) throw new Error(`Google API failed to generate a key.`)
|
||||
return key
|
||||
}
|
||||
|
||||
|
||||
const newOpenAICompatibleSDK = async ({ settingsOfProvider, providerName, includeInPayload }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName, includeInPayload?: { [s: string]: any } }) => {
|
||||
const commonPayloadOpts: ClientOptions = {
|
||||
dangerouslyAllowBrowser: true,
|
||||
...includeInPayload,
|
||||
|
|
@ -56,6 +68,14 @@ const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPay
|
|||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'liteLLM') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'lmStudio') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'openRouter') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({
|
||||
|
|
@ -70,8 +90,22 @@ const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPay
|
|||
}
|
||||
else if (providerName === 'gemini') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai/', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
return new OpenAI({ baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'googleVertex') {
|
||||
// https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library
|
||||
const apiKey = await getGoogleApiKey()
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
const baseURL = `https://${thisConfig.region}-aiplatform.googleapis.com/v1/projects/${thisConfig.project}/locations/${thisConfig.region}/endpoints/${'openapi'}`
|
||||
return new OpenAI({ baseURL: baseURL, apiKey: apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'microsoftAzure') {
|
||||
// https://learn.microsoft.com/en-us/rest/api/aifoundry/model-inference/get-chat-completions/get-chat-completions?view=rest-aifoundry-model-inference-2024-05-01-preview&tabs=HTTP
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
const baseURL = `https://${thisConfig.project}.services.ai.azure.com/api/models/chat/completions??api-version=${thisConfig.azureApiVersion}`
|
||||
return new OpenAI({ baseURL: baseURL, apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
|
||||
else if (providerName === 'deepseek') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
|
|
@ -97,7 +131,7 @@ const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPay
|
|||
}
|
||||
|
||||
|
||||
const _sendOpenAICompatibleFIM = ({ messages: { prefix, suffix, stopTokens }, onFinalMessage, onError, settingsOfProvider, modelName: modelName_, _setAborter, providerName, }: SendFIMParams_Internal) => {
|
||||
const _sendOpenAICompatibleFIM = async ({ messages: { prefix, suffix, stopTokens }, onFinalMessage, onError, settingsOfProvider, modelName: modelName_, _setAborter, providerName, }: SendFIMParams_Internal) => {
|
||||
const { modelName, supportsFIM } = getModelCapabilities(providerName, modelName_)
|
||||
if (!supportsFIM) {
|
||||
if (modelName === modelName_)
|
||||
|
|
@ -107,7 +141,7 @@ const _sendOpenAICompatibleFIM = ({ messages: { prefix, suffix, stopTokens }, on
|
|||
return
|
||||
}
|
||||
|
||||
const openai = newOpenAICompatibleSDK({ providerName, settingsOfProvider })
|
||||
const openai = await newOpenAICompatibleSDK({ providerName, settingsOfProvider })
|
||||
openai.completions
|
||||
.create({
|
||||
model: modelName,
|
||||
|
|
@ -178,7 +212,7 @@ const openAIToolToRawToolCallObj = (name: string, toolParamsStr: string, id: str
|
|||
// ------------ OPENAI-COMPATIBLE ------------
|
||||
|
||||
|
||||
const _sendOpenAICompatibleChat = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName: modelName_, _setAborter, providerName, chatMode, separateSystemMessage }: SendChatParams_Internal) => {
|
||||
const _sendOpenAICompatibleChat = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName: modelName_, _setAborter, providerName, chatMode, separateSystemMessage }: SendChatParams_Internal) => {
|
||||
const {
|
||||
modelName,
|
||||
specialToolFormat,
|
||||
|
|
@ -199,7 +233,7 @@ const _sendOpenAICompatibleChat = ({ messages, onText, onFinalMessage, onError,
|
|||
: {}
|
||||
|
||||
// instance
|
||||
const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload })
|
||||
const openai: OpenAI = await newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload })
|
||||
const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
|
||||
model: modelName,
|
||||
messages: messages as any,
|
||||
|
|
@ -300,7 +334,7 @@ const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: onError_,
|
|||
onError_({ error })
|
||||
}
|
||||
try {
|
||||
const openai = newOpenAICompatibleSDK({ providerName, settingsOfProvider })
|
||||
const openai = await newOpenAICompatibleSDK({ providerName, settingsOfProvider })
|
||||
openai.models.list()
|
||||
.then(async (response) => {
|
||||
const models: OpenAIModel[] = []
|
||||
|
|
@ -360,7 +394,7 @@ const anthropicToolToRawToolCallObj = (toolBlock: Anthropic.Messages.ToolUseBloc
|
|||
}
|
||||
|
||||
// ------------ ANTHROPIC ------------
|
||||
const sendAnthropicChat = ({ messages, providerName, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName: modelName_, _setAborter, separateSystemMessage, chatMode }: SendChatParams_Internal) => {
|
||||
const sendAnthropicChat = async ({ messages, providerName, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName: modelName_, _setAborter, separateSystemMessage, chatMode }: SendChatParams_Internal) => {
|
||||
const {
|
||||
modelName,
|
||||
specialToolFormat,
|
||||
|
|
@ -584,7 +618,7 @@ const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfProvider,
|
|||
|
||||
type CallFnOfProvider = {
|
||||
[providerName in ProviderName]: {
|
||||
sendChat: (params: SendChatParams_Internal) => void;
|
||||
sendChat: (params: SendChatParams_Internal) => Promise<void>;
|
||||
sendFIM: ((params: SendFIMParams_Internal) => void) | null;
|
||||
list: ((params: ListParams_Internal<any>) => void) | null;
|
||||
}
|
||||
|
|
@ -646,6 +680,27 @@ export const sendLLMMessageToProviderImplementation = {
|
|||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
|
||||
lmStudio: {
|
||||
sendChat: (params) => _sendOpenAICompatibleChat(params),
|
||||
sendFIM: null, // lmStudio has no suffix parameter in /completions
|
||||
list: null,
|
||||
},
|
||||
liteLLM: {
|
||||
sendChat: (params) => _sendOpenAICompatibleChat(params),
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
googleVertex: {
|
||||
sendChat: (params) => _sendOpenAICompatibleChat(params),
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
microsoftAzure: {
|
||||
sendChat: (params) => _sendOpenAICompatibleChat(params),
|
||||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
} satisfies CallFnOfProvider
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { displayInfoOfProviderName } from '../../common/voidSettingsTypes.js';
|
|||
import { sendLLMMessageToProviderImplementation } from './sendLLMMessage.impl.js';
|
||||
|
||||
|
||||
export const sendLLMMessage = ({
|
||||
export const sendLLMMessage = async ({
|
||||
messagesType,
|
||||
messages: messages_,
|
||||
onText: onText_,
|
||||
|
|
@ -108,12 +108,12 @@ export const sendLLMMessage = ({
|
|||
}
|
||||
const { sendFIM, sendChat } = implementation
|
||||
if (messagesType === 'chatMessages') {
|
||||
sendChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName, _setAborter, providerName, separateSystemMessage, chatMode })
|
||||
await sendChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName, _setAborter, providerName, separateSystemMessage, chatMode })
|
||||
return
|
||||
}
|
||||
if (messagesType === 'FIMMessage') {
|
||||
if (sendFIM) {
|
||||
sendFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName, _setAborter, providerName, separateSystemMessage })
|
||||
await sendFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName, _setAborter, providerName, separateSystemMessage })
|
||||
return
|
||||
}
|
||||
onError({ message: `Error: This provider does not support Autocomplete yet.`, fullError: null })
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ export class LLMMessageChannel implements IServerChannel {
|
|||
}
|
||||
|
||||
// aborters for above
|
||||
private readonly abortRefOfRequestId: Record<string, AbortRef> = {}
|
||||
private readonly _infoOfRunningRequest: Record<string, { waitForSend: Promise<void> | undefined, abortRef: AbortRef }> = {}
|
||||
|
||||
|
||||
// list
|
||||
|
|
@ -72,7 +72,7 @@ export class LLMMessageChannel implements IServerChannel {
|
|||
this._callSendLLMMessage(params)
|
||||
}
|
||||
else if (command === 'abort') {
|
||||
this._callAbort(params)
|
||||
await this._callAbort(params)
|
||||
}
|
||||
else if (command === 'ollamaList') {
|
||||
this._callOllamaList(params)
|
||||
|
|
@ -93,24 +93,27 @@ export class LLMMessageChannel implements IServerChannel {
|
|||
private async _callSendLLMMessage(params: MainSendLLMMessageParams) {
|
||||
const { requestId } = params;
|
||||
|
||||
if (!(requestId in this.abortRefOfRequestId))
|
||||
this.abortRefOfRequestId[requestId] = { current: null }
|
||||
if (!(requestId in this._infoOfRunningRequest))
|
||||
this._infoOfRunningRequest[requestId] = { waitForSend: undefined, abortRef: { current: null } }
|
||||
|
||||
const mainThreadParams: SendLLMMessageParams = {
|
||||
...params,
|
||||
onText: (p) => { this.llmMessageEmitters.onText.fire({ requestId, ...p }); },
|
||||
onFinalMessage: (p) => { this.llmMessageEmitters.onFinalMessage.fire({ requestId, ...p }); },
|
||||
onError: (p) => { console.log('sendLLM: firing err'); this.llmMessageEmitters.onError.fire({ requestId, ...p }); },
|
||||
abortRef: this.abortRefOfRequestId[requestId],
|
||||
abortRef: this._infoOfRunningRequest[requestId].abortRef,
|
||||
}
|
||||
sendLLMMessage(mainThreadParams, this.metricsService);
|
||||
const p = sendLLMMessage(mainThreadParams, this.metricsService);
|
||||
this._infoOfRunningRequest[requestId].waitForSend = p
|
||||
}
|
||||
|
||||
private _callAbort(params: MainLLMMessageAbortParams) {
|
||||
private async _callAbort(params: MainLLMMessageAbortParams) {
|
||||
const { requestId } = params;
|
||||
if (!(requestId in this.abortRefOfRequestId)) return
|
||||
this.abortRefOfRequestId[requestId].current?.()
|
||||
delete this.abortRefOfRequestId[requestId]
|
||||
if (!(requestId in this._infoOfRunningRequest)) return
|
||||
const { waitForSend, abortRef } = this._infoOfRunningRequest[requestId]
|
||||
await waitForSend // wait for the send to finish so we know abortRef was set
|
||||
abortRef?.current?.()
|
||||
delete this._infoOfRunningRequest[requestId]
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue