diff --git a/package-lock.json b/package-lock.json index d7b11ce9..bfa533d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,6 +45,7 @@ "cross-spawn": "^7.0.6", "diff": "^7.0.0", "eslint-plugin-react": "^7.37.4", + "google-auth-library": "^9.15.1", "groq-sdk": "^0.15.0", "http-proxy-agent": "^7.0.0", "https-proxy-agent": "^7.0.2", @@ -5991,6 +5992,15 @@ "node": "*" } }, + "node_modules/bignumber.js": { + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.2.1.tgz", + "integrity": "sha512-+NzaKgOUvInq9TIUZ1+DRspzf/HApkCwD4btfuasFTdrfnOxqx853TgDpMolp+uv4RpRp7bPcEU2zKr9+fRmyw==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -6221,6 +6231,12 @@ "node": ">=0.4.0" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -8068,6 +8084,15 @@ "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", "dev": true }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/editorconfig": { "version": "0.15.2", "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.2.tgz", @@ -9284,8 +9309,7 @@ "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "dev": true + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "node_modules/extend-shallow": { "version": "3.0.2", @@ -10205,6 +10229,68 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gaxios/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", + "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.1.1", + "google-logging-utils": "^0.0.2", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -10841,6 +10927,32 @@ "node": ">= 0.10" } }, + "node_modules/google-auth-library": { + "version": "9.15.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", + "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-logging-utils": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", + "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -10913,6 +11025,19 @@ "undici-types": "~5.26.4" } }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/gulp": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/gulp/-/gulp-4.0.2.tgz", @@ -13873,6 +13998,15 @@ "node": ">=6" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -13993,6 +14127,27 @@ "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", "dev": true }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/kerberos": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/kerberos/-/kerberos-2.1.1.tgz", diff --git a/package.json b/package.json index 6e65b143..0b61062a 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ "cross-spawn": "^7.0.6", "diff": "^7.0.0", "eslint-plugin-react": "^7.37.4", + "google-auth-library": "^9.15.1", "groq-sdk": "^0.15.0", "http-proxy-agent": "^7.0.0", "https-proxy-agent": "^7.0.2", diff --git a/src/vs/workbench/contrib/void/browser/react/src/util/inputs.tsx b/src/vs/workbench/contrib/void/browser/react/src/util/inputs.tsx index 917ef5a0..0b2ce3a8 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/util/inputs.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/util/inputs.tsx @@ -709,7 +709,7 @@ export const VoidCustomDropdownBox = >({ )} - + {optionName} {optionDetail} diff --git a/src/vs/workbench/contrib/void/browser/react/src/void-onboarding/VoidOnboarding.tsx b/src/vs/workbench/contrib/void/browser/react/src/void-onboarding/VoidOnboarding.tsx index 40ee767c..d61ca850 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/void-onboarding/VoidOnboarding.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/void-onboarding/VoidOnboarding.tsx @@ -497,7 +497,7 @@ const VoidOnboardingContent = () => { const providerNamesOfWantToUseOption: { [wantToUseOption in WantToUseOption]: ProviderName[] } = { smart: ['anthropic', 'openAI', 'gemini', 'openRouter'], - private: ['ollama', 'vLLM', 'openAICompatible'], + private: ['ollama', 'vLLM', 'openAICompatible', 'lmStudio'], cheap: ['gemini', 'deepseek', 'openRouter', 'ollama', 'vLLM'], all: providerNames, } diff --git a/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/Settings.tsx b/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/Settings.tsx index 810def44..a8cbe365 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/Settings.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/Settings.tsx @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------*/ import React, { useCallback, useEffect, useMemo, useState } from 'react' -import { ProviderName, SettingName, displayInfoOfSettingName, providerNames, VoidStatefulModelInfo, customSettingNamesOfProvider, RefreshableProviderName, refreshableProviderNames, displayInfoOfProviderName, nonlocalProviderNames, localProviderNames, GlobalSettingName, featureNames, displayInfoOfFeatureName, isProviderNameDisabled, FeatureName, hasDownloadButtonsOnModelsProviderNames } from '../../../../common/voidSettingsTypes.js' +import { ProviderName, SettingName, displayInfoOfSettingName, providerNames, VoidStatefulModelInfo, customSettingNamesOfProvider, RefreshableProviderName, refreshableProviderNames, displayInfoOfProviderName, nonlocalProviderNames, localProviderNames, GlobalSettingName, featureNames, displayInfoOfFeatureName, isProviderNameDisabled, FeatureName, hasDownloadButtonsOnModelsProviderNames, subTextMdOfProviderName } from '../../../../common/voidSettingsTypes.js' import ErrorBoundary from '../sidebar-tsx/ErrorBoundary.js' import { VoidButtonBgDarken, VoidCustomDropdownBox, VoidInputBox2, VoidSimpleInputBox, VoidSwitch } from '../util/inputs.js' import { useAccessor, useIsDark, useRefreshModelListener, useRefreshModelState, useSettingsState } from '../util/services.js' @@ -344,9 +344,9 @@ export const ModelDump = () => { // providers -const ProviderSetting = ({ providerName, settingName }: { providerName: ProviderName, settingName: SettingName }) => { +const ProviderSetting = ({ providerName, settingName, subTextMd }: { providerName: ProviderName, settingName: SettingName, subTextMd: React.ReactNode }) => { - const { title: settingTitle, placeholder, isPasswordField, subTextMd } = displayInfoOfSettingName(providerName, settingName) + const { title: settingTitle, placeholder, isPasswordField } = displayInfoOfSettingName(providerName, settingName) const accessor = useAccessor() const voidSettingsService = accessor.get('IVoidSettingsService') @@ -370,10 +370,9 @@ const ProviderSetting = ({ providerName, settingName }: { providerName: Provider passwordBlur={isPasswordField} compact={true} /> - {subTextMd === undefined ? null :
- + {!subTextMd ? null :
+ {subTextMd}
} -
} @@ -456,7 +455,14 @@ export const SettingsForProvider = ({ providerName, showProviderTitle, showProvi
{/* settings besides models (e.g. api key) */} {settingNames.map((settingName, i) => { - return + + return } + /> })} {showProviderSuggestions && needsModel ? diff --git a/src/vs/workbench/contrib/void/common/modelCapabilities.ts b/src/vs/workbench/contrib/void/common/modelCapabilities.ts index f75d5549..9e142fa4 100644 --- a/src/vs/workbench/contrib/void/common/modelCapabilities.ts +++ b/src/vs/workbench/contrib/void/common/modelCapabilities.ts @@ -43,7 +43,22 @@ export const defaultProviderSettings = { }, mistral: { apiKey: '', - } + }, + lmStudio: { + endpoint: 'http://localhost:1234', + }, + liteLLM: { // https://docs.litellm.ai/docs/providers/openai_compatible + endpoint: '', + }, + googleVertex: { // google https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library + region: 'us-west2', + project: '', + }, + microsoftAzure: { // microsoft Azure Foundry + project: '', // really 'resource' + apiKey: '', + azureApiVersion: '2024-05-01-preview', + }, } as const @@ -84,6 +99,8 @@ export const defaultModelsOfProvider = { ], vLLM: [ // autodetected ], + lmStudio: [], // autodetected + openRouter: [ // https://openrouter.ai/models // 'anthropic/claude-3.7-sonnet:thinking', 'anthropic/claude-3.7-sonnet', @@ -112,6 +129,11 @@ export const defaultModelsOfProvider = { 'ministral-8b-latest', ], openAICompatible: [], // fallback + googleVertex: [], + microsoftAzure: [], + liteLLM: [], + + } as const satisfies Record @@ -618,6 +640,7 @@ const geminiModelOptions = { // https://ai.google.dev/gemini-api/docs/pricing downloadable: false, supportsFIM: false, supportsSystemMessage: 'system-role', + specialToolFormat: 'openai-style', reasoningCapabilities: false, }, 'gemini-2.0-flash': { @@ -806,6 +829,25 @@ const groqSettings: VoidStaticProviderInfo = { modelOptionsFallback: (modelName) => { return null } } + +// ---------------- GOOGLE VERTEX ---------------- +const googleVertexModelOptions = { +} as const satisfies Record +const googleVertexSettings: VoidStaticProviderInfo = { + modelOptions: googleVertexModelOptions, + modelOptionsFallback: (modelName) => { return null } +} + +// ---------------- MICROSOFT AZURE ---------------- +const microsoftAzureModelOptions = { +} as const satisfies Record +const microsoftAzureSettings: VoidStaticProviderInfo = { + modelOptions: microsoftAzureModelOptions, + modelOptionsFallback: (modelName) => { return null } +} + + +// ---------------- VLLM, OLLAMA, OPENAICOMPAT (self-hosted / local) ---------------- const ollamaModelOptions = { 'qwen2.5-coder:1.5b': { contextWindow: 32_000, @@ -858,9 +900,6 @@ const ollamaModelOptions = { export const ollamaRecommendedModels = ['qwen2.5-coder:1.5b', 'llama3.1', 'qwq', 'deepseek-r1'] as const satisfies (keyof typeof ollamaModelOptions)[] - -// ---------------- VLLM, OLLAMA, OPENAICOMPAT (self-hosted / local) ---------------- - const vLLMSettings: VoidStaticProviderInfo = { // reasoning: OAICompat + response.choices[0].delta.reasoning_content // https://docs.vllm.ai/en/stable/features/reasoning_outputs.html#streaming-chat-completions providerReasoningIOSettings: { output: { nameOfFieldInDelta: 'reasoning_content' }, }, @@ -868,6 +907,18 @@ const vLLMSettings: VoidStaticProviderInfo = { modelOptions: {}, // TODO } +const lmStudioSettings: VoidStaticProviderInfo = { + providerReasoningIOSettings: { output: { nameOfFieldInDelta: 'reasoning_content' }, }, + modelOptionsFallback: (modelName) => extensiveModelFallback(modelName, { downloadable: { sizeGb: 'not-known' } }), + modelOptions: {}, // TODO +} + +const liteLLMSettings: VoidStaticProviderInfo = { + providerReasoningIOSettings: { output: { nameOfFieldInDelta: 'reasoning_content' }, }, + modelOptionsFallback: (modelName) => extensiveModelFallback(modelName, { downloadable: { sizeGb: 'not-known' } }), + modelOptions: {}, // TODO +} + const ollamaSettings: VoidStaticProviderInfo = { // reasoning: we need to filter out reasoning tags manually providerReasoningIOSettings: { output: { needsManualParse: true }, }, @@ -1027,9 +1078,12 @@ const modelSettingsOfProvider: { [providerName in ProviderName]: VoidStaticProvi ollama: ollamaSettings, openAICompatible: openaiCompatible, mistral: mistralSettings, - // googleVertex: {}, - // microsoftAzure: {}, - // openHands: {}, + + liteLLM: liteLLMSettings, + lmStudio: lmStudioSettings, + + googleVertex: googleVertexSettings, + microsoftAzure: microsoftAzureSettings, } as const diff --git a/src/vs/workbench/contrib/void/common/prompt/prompts.ts b/src/vs/workbench/contrib/void/common/prompt/prompts.ts index fc2c554b..61372d99 100644 --- a/src/vs/workbench/contrib/void/common/prompt/prompts.ts +++ b/src/vs/workbench/contrib/void/common/prompt/prompts.ts @@ -330,6 +330,7 @@ Here's an example of a good edit suggestion: ${fileNameEditExample}.`) } + details.push(`NEVER write the FULL PATH of a file when speaking with the user. Just write the file name ONLY.`) details.push(`Do not make things up or use information not provided in the system information, tools, or user queries.`) details.push(`Today's date is ${new Date().toDateString()}.`) @@ -459,9 +460,9 @@ ${tripleTick[1]} 1. The change to make will be labeled \`CHANGE\` and the original file will be labeled \`ORIGINAL_FILE\`. -2. You are allowed to output multiple SEARCH/REPLACE blocks. +2. Your SEARCH/REPLACE block(s) must implement the change EXACTLY. Do not introduce (or omit) any new comments, spaces, or whitespace. -3. Your SEARCH/REPLACE block(s) must implement the change EXACTLY. Do not introduce (or omit) any new comments, spaces, or whitespace. +3. You are allowed to output multiple SEARCH/REPLACE blocks. 4. Your output should consist ONLY of SEARCH/REPLACE blocks. Do NOT output any text or explanations before or after this. diff --git a/src/vs/workbench/contrib/void/common/refreshModelService.ts b/src/vs/workbench/contrib/void/common/refreshModelService.ts index 8123441d..8f64beae 100644 --- a/src/vs/workbench/contrib/void/common/refreshModelService.ts +++ b/src/vs/workbench/contrib/void/common/refreshModelService.ts @@ -8,7 +8,7 @@ import { ILLMMessageService } from './sendLLMMessageService.js'; import { Emitter, Event } from '../../../../base/common/event.js'; import { Disposable, IDisposable } from '../../../../base/common/lifecycle.js'; import { RefreshableProviderName, refreshableProviderNames, SettingsOfProvider } from './voidSettingsTypes.js'; -import { OllamaModelResponse, VLLMModelResponse } from './sendLLMMessageTypes.js'; +import { LMStudioModelResponse, OllamaModelResponse, VLLMModelResponse } from './sendLLMMessageTypes.js'; import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js'; import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; @@ -46,6 +46,7 @@ export type RefreshModelStateOfProvider = Record { if (providerName === 'ollama') return (model as OllamaModelResponse).name; else if (providerName === 'vLLM') return (model as VLLMModelResponse).id; + else if (providerName === 'lmStudio') return (model as LMStudioModelResponse).id; else throw new Error('refreshMode fn: unknown provider', providerName); }), { enableProviderOnSuccess: options.enableProviderOnSuccess, hideRefresh: options.doNotFire } diff --git a/src/vs/workbench/contrib/void/common/sendLLMMessageTypes.ts b/src/vs/workbench/contrib/void/common/sendLLMMessageTypes.ts index a345fde0..975e6de5 100644 --- a/src/vs/workbench/contrib/void/common/sendLLMMessageTypes.ts +++ b/src/vs/workbench/contrib/void/common/sendLLMMessageTypes.ts @@ -170,6 +170,7 @@ type OpenaiCompatibleModelResponse = { } export type VLLMModelResponse = OpenaiCompatibleModelResponse +export type LMStudioModelResponse = OpenaiCompatibleModelResponse diff --git a/src/vs/workbench/contrib/void/common/voidSettingsService.ts b/src/vs/workbench/contrib/void/common/voidSettingsService.ts index 819a01b7..40de78a2 100644 --- a/src/vs/workbench/contrib/void/common/voidSettingsService.ts +++ b/src/vs/workbench/contrib/void/common/voidSettingsService.ts @@ -245,17 +245,36 @@ class VoidSettingsService extends Disposable implements IVoidSettingsService { } // the stored data structure might be outdated, so we need to update it here - readS = { - ...readS, - settingsOfProvider: { + try { + readS = { + ...readS, ...defaultSettingsOfProvider, ...readS.settingsOfProvider, - mistral: { // we added mistral - ...defaultSettingsOfProvider.mistral, - ...readS.settingsOfProvider.mistral, - }, - } // we added mistral + } + + for (const providerName of providerNames) { + readS.settingsOfProvider[providerName] = { + ...defaultSettingsOfProvider[providerName], + ...readS.settingsOfProvider[providerName], + } as any + } + // readS = { + // ...readS, + // settingsOfProvider: { + // ...defaultSettingsOfProvider, + // ...readS.settingsOfProvider, + // mistral: { // we added mistral + // ...defaultSettingsOfProvider.mistral, + // ...readS.settingsOfProvider.mistral, + // }, + // } // we added mistral + // } } + + catch (e) { + readS = defaultState() + } + this.state = readS this.state = _stateWithUpdatedDefaultModels(this.state) this.state = _validatedModelState(this.state); @@ -263,6 +282,7 @@ class VoidSettingsService extends Disposable implements IVoidSettingsService { this._resolver(); this._onDidChangeState.fire(); + } diff --git a/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts b/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts index 3977f31b..cf026f8c 100644 --- a/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts +++ b/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts @@ -15,7 +15,7 @@ type UnionOfKeys = T extends T ? keyof T : never; export type ProviderName = keyof typeof defaultProviderSettings export const providerNames = Object.keys(defaultProviderSettings) as ProviderName[] -export const localProviderNames = ['ollama', 'vLLM'] satisfies ProviderName[] // all local names +export const localProviderNames = ['ollama', 'vLLM', 'lmStudio'] satisfies ProviderName[] // all local names export const nonlocalProviderNames = providerNames.filter((name) => !(localProviderNames as string[]).includes(name)) // all non-local names type CustomSettingName = UnionOfKeys @@ -59,74 +59,78 @@ type DisplayInfoForProviderName = { export const displayInfoOfProviderName = (providerName: ProviderName): DisplayInfoForProviderName => { if (providerName === 'anthropic') { - return { - title: 'Anthropic', - } + return { title: 'Anthropic', } } else if (providerName === 'openAI') { - return { - title: 'OpenAI', - } + return { title: 'OpenAI', } } else if (providerName === 'deepseek') { - return { - // title: 'DeepSeek.com API', - title: 'DeepSeek', - } + return { title: 'DeepSeek', } } else if (providerName === 'openRouter') { - return { - title: 'OpenRouter', - } + return { title: 'OpenRouter', } } else if (providerName === 'ollama') { - return { - title: 'Ollama', - } + return { title: 'Ollama', } } else if (providerName === 'vLLM') { - return { - title: 'vLLM', - } + return { title: 'vLLM', } + } + else if (providerName === 'liteLLM') { + return { title: 'LiteLLM', } + } + else if (providerName === 'lmStudio') { + return { title: 'LM Studio', } } else if (providerName === 'openAICompatible') { - return { - title: 'OpenAI-Compatible', - } + return { title: 'OpenAI-Compatible', } } else if (providerName === 'gemini') { - return { - // title: 'Gemini API', - title: 'Gemini', - } + return { title: 'Gemini', } } else if (providerName === 'groq') { - return { - // title: 'Groq.com API', - title: 'Groq', - } + return { title: 'Groq', } } else if (providerName === 'xAI') { - return { - // title: 'Grok (xAI)', - title: 'xAI', - } + return { title: 'xAI', } } else if (providerName === 'mistral') { - return { - // title: 'Mistral API', - title: 'Mistral', - } + return { title: 'Mistral', } + } + else if (providerName === 'googleVertex') { + return { title: 'Google Vertex AI', } + } + else if (providerName === 'microsoftAzure') { + return { title: 'Microsoft Azure OpenAI', } } - throw new Error(`descOfProviderName: Unknown provider name: "${providerName}"`) } +export const subTextMdOfProviderName = (providerName: ProviderName): string => { + + if (providerName === 'anthropic') return 'Get your [API Key here](https://console.anthropic.com/settings/keys).' + if (providerName === 'openAI') return 'Get your [API Key here](https://platform.openai.com/api-keys).' + if (providerName === 'deepseek') return 'Get your [API Key here](https://platform.deepseek.com/api_keys).' + if (providerName === 'openRouter') return 'Get your [API Key here](https://openrouter.ai/settings/keys).' + if (providerName === 'gemini') return 'Get your [API Key here](https://aistudio.google.com/apikey).' + if (providerName === 'groq') return 'Get your [API Key here](https://console.groq.com/keys).' + if (providerName === 'xAI') return 'Get your [API Key here](https://console.x.ai).' + if (providerName === 'mistral') return 'Get your [API Key here](https://console.mistral.ai/api-keys).' + if (providerName === 'openAICompatible') return `Use any OpenAI-compatible endpoint (LM Studio, LiteLM, etc).` + if (providerName === 'googleVertex') return 'You must authenticate before using Vertex with Void. Read more about endpoints [here](https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library), and regions [here](https://cloud.google.com/vertex-ai/docs/general/locations#available-regions).' + if (providerName === 'microsoftAzure') return 'Read more about endpoints [here](https://learn.microsoft.com/en-us/rest/api/aifoundry/model-inference/get-chat-completions/get-chat-completions?view=rest-aifoundry-model-inference-2024-05-01-preview&tabs=HTTP), and get your API key [here](https://learn.microsoft.com/en-us/azure/search/search-security-api-keys?tabs=rest-use%2Cportal-find%2Cportal-query#find-existing-keys).' + if (providerName === 'ollama') return 'If you would like to change this endpoint, please read more about [Endpoints here](https://github.com/ollama/ollama/blob/main/docs/faq.md#how-can-i-expose-ollama-on-my-network).' + if (providerName === 'vLLM') return 'If you would like to change this endpoint, please read more about [Endpoints here](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server).' + if (providerName === 'lmStudio') return 'If you would like to change this endpoint, please more about [Endpoints here](https://lmstudio.ai/docs/app/api/endpoints/openai).' + if (providerName === 'liteLLM') return 'Read more about endpoints [here](https://docs.litellm.ai/docs/providers/openai_compatible).' + + throw new Error(`subTextMdOfProviderName: Unknown provider name: "${providerName}"`) +} + type DisplayInfo = { title: string; placeholder: string; - subTextMd?: string; isPasswordField?: boolean; } export const displayInfoOfSettingName = (providerName: ProviderName, settingName: SettingName): DisplayInfo => { @@ -140,23 +144,14 @@ export const displayInfoOfSettingName = (providerName: ProviderName, settingName providerName === 'openAI' ? 'sk-proj-key...' : providerName === 'deepseek' ? 'sk-key...' : providerName === 'openRouter' ? 'sk-or-key...' : // sk-or-v1-key - providerName === 'gemini' ? 'key...' : + providerName === 'gemini' ? 'AIzaSy...' : providerName === 'groq' ? 'gsk_key...' : providerName === 'openAICompatible' ? 'sk-key...' : providerName === 'xAI' ? 'xai-key...' : providerName === 'mistral' ? 'api-key...' : - '', + providerName === 'googleVertex' ? 'AIzaSy...' : + '', - subTextMd: providerName === 'anthropic' ? 'Get your [API Key here](https://console.anthropic.com/settings/keys).' : - providerName === 'openAI' ? 'Get your [API Key here](https://platform.openai.com/api-keys).' : - providerName === 'deepseek' ? 'Get your [API Key here](https://platform.deepseek.com/api_keys).' : - providerName === 'openRouter' ? 'Get your [API Key here](https://openrouter.ai/settings/keys).' : - providerName === 'gemini' ? 'Get your [API Key here](https://aistudio.google.com/apikey).' : - providerName === 'groq' ? 'Get your [API Key here](https://console.groq.com/keys).' : - providerName === 'xAI' ? 'Get your [API Key here](https://console.x.ai).' : - providerName === 'mistral' ? 'Get your [API Key here](https://console.mistral.ai/api-keys).' : - providerName === 'openAICompatible' ? `Use any OpenAI-compatible endpoint (LM Studio, LiteLM, etc).` : - '', isPasswordField: true, } } @@ -164,19 +159,51 @@ export const displayInfoOfSettingName = (providerName: ProviderName, settingName return { title: providerName === 'ollama' ? 'Endpoint' : providerName === 'vLLM' ? 'Endpoint' : - providerName === 'openAICompatible' ? 'baseURL' : // (do not include /chat/completions) - '(never)', + providerName === 'lmStudio' ? 'Endpoint' : + providerName === 'openAICompatible' ? 'baseURL' : // (do not include /chat/completions) + providerName === 'googleVertex' ? 'baseURL' : + providerName === 'microsoftAzure' ? 'baseURL' : + providerName === 'liteLLM' ? 'baseURL' : + '(never)', placeholder: providerName === 'ollama' ? defaultProviderSettings.ollama.endpoint : providerName === 'vLLM' ? defaultProviderSettings.vLLM.endpoint : providerName === 'openAICompatible' ? 'https://my-website.com/v1' - : '(never)', + : providerName === 'lmStudio' ? defaultProviderSettings.lmStudio.endpoint + : providerName === 'liteLLM' ? 'http://localhost:4000' + : '(never)', + - subTextMd: providerName === 'ollama' ? 'If you would like to change this endpoint, please read more about [Endpoints here](https://github.com/ollama/ollama/blob/main/docs/faq.md#how-can-i-expose-ollama-on-my-network).' : - providerName === 'vLLM' ? 'If you would like to change this endpoint, please read more about [Endpoints here](https://docs.vllm.ai/en/latest/getting_started/quickstart.html#openai-compatible-server).' : - undefined, } } + else if (settingName === 'region') { + // vertex only + return { + title: 'Region', + placeholder: providerName === 'googleVertex' ? defaultProviderSettings.googleVertex.region + : '' + } + } + else if (settingName === 'azureApiVersion') { + // azure only + return { + title: 'API Version', + placeholder: providerName === 'microsoftAzure' ? defaultProviderSettings.microsoftAzure.azureApiVersion + : '' + } + } + else if (settingName === 'project') { + return { + title: providerName === 'googleVertex' ? 'Project' + : providerName === 'microsoftAzure' ? 'Resource' + : '', + placeholder: providerName === 'googleVertex' ? 'my-project' + : providerName === 'microsoftAzure' ? 'my-resource' + : '' + + } + + } else if (settingName === '_didFillInProviderSettings') { return { title: '(never)', @@ -200,6 +227,9 @@ export const displayInfoOfSettingName = (providerName: ProviderName, settingName const defaultCustomSettings: Record = { apiKey: undefined, endpoint: undefined, + region: undefined, + project: undefined, + azureApiVersion: undefined, } @@ -252,6 +282,18 @@ export const defaultSettingsOfProvider: SettingsOfProvider = { ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.mistral), _didFillInProviderSettings: undefined, }, + liteLLM: { + ...defaultCustomSettings, + ...defaultProviderSettings.liteLLM, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.liteLLM), + _didFillInProviderSettings: undefined, + }, + lmStudio: { + ...defaultCustomSettings, + ...defaultProviderSettings.lmStudio, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.lmStudio), + _didFillInProviderSettings: undefined, + }, groq: { // aggregator (serves models from multiple providers) ...defaultCustomSettings, ...defaultProviderSettings.groq, @@ -282,6 +324,18 @@ export const defaultSettingsOfProvider: SettingsOfProvider = { ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.vLLM), _didFillInProviderSettings: undefined, }, + googleVertex: { // aggregator (serves models from multiple providers) + ...defaultCustomSettings, + ...defaultProviderSettings.googleVertex, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.googleVertex), + _didFillInProviderSettings: undefined, + }, + microsoftAzure: { // aggregator (serves models from multiple providers) + ...defaultCustomSettings, + ...defaultProviderSettings.microsoftAzure, + ...modelInfoOfDefaultModelNames(defaultModelsOfProvider.microsoftAzure), + _didFillInProviderSettings: undefined, + }, } diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.impl.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.impl.ts index 1645cdac..ea4f70c9 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.impl.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.impl.ts @@ -10,6 +10,7 @@ import { Ollama } from 'ollama'; import OpenAI, { ClientOptions } from 'openai'; import { MistralCore } from '@mistralai/mistralai/core.js'; import { fimComplete } from '@mistralai/mistralai/funcs/fimComplete.js'; +import { GoogleAuth } from 'google-auth-library' /* eslint-enable */ import { AnthropicLLMChatMessage, LLMChatMessage, LLMFIMMessage, ModelListParams, OllamaModelResponse, OnError, OnFinalMessage, OnText, RawToolCallObj, RawToolParamsObj } from '../../common/sendLLMMessageTypes.js'; @@ -19,6 +20,8 @@ import { extractReasoningWrapper, extractXMLToolsWrapper } from './extractGramma import { availableTools, InternalToolInfo, isAToolName, ToolParamName, voidTools } from '../../common/prompt/prompts.js'; + + type InternalCommonMessageParams = { onText: OnText; onFinalMessage: OnFinalMessage; @@ -39,7 +42,16 @@ const invalidApiKeyMessage = (providerName: ProviderName) => `Invalid ${displayI // ------------ OPENAI-COMPATIBLE (HELPERS) ------------ -const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPayload }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName, includeInPayload?: { [s: string]: any } }) => { +const getGoogleApiKey = async () => { + // module‑level singleton + const auth = new GoogleAuth({ scopes: `https://www.googleapis.com/auth/cloud-platform` }); + const key = await auth.getAccessToken() + if (!key) throw new Error(`Google API failed to generate a key.`) + return key +} + + +const newOpenAICompatibleSDK = async ({ settingsOfProvider, providerName, includeInPayload }: { settingsOfProvider: SettingsOfProvider, providerName: ProviderName, includeInPayload?: { [s: string]: any } }) => { const commonPayloadOpts: ClientOptions = { dangerouslyAllowBrowser: true, ...includeInPayload, @@ -56,6 +68,14 @@ const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPay const thisConfig = settingsOfProvider[providerName] return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts }) } + else if (providerName === 'liteLLM') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts }) + } + else if (providerName === 'lmStudio') { + const thisConfig = settingsOfProvider[providerName] + return new OpenAI({ baseURL: `${thisConfig.endpoint}/v1`, apiKey: 'noop', ...commonPayloadOpts }) + } else if (providerName === 'openRouter') { const thisConfig = settingsOfProvider[providerName] return new OpenAI({ @@ -70,8 +90,22 @@ const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPay } else if (providerName === 'gemini') { const thisConfig = settingsOfProvider[providerName] - return new OpenAI({ baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai/', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + return new OpenAI({ baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) } + else if (providerName === 'googleVertex') { + // https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library + const apiKey = await getGoogleApiKey() + const thisConfig = settingsOfProvider[providerName] + const baseURL = `https://${thisConfig.region}-aiplatform.googleapis.com/v1/projects/${thisConfig.project}/locations/${thisConfig.region}/endpoints/${'openapi'}` + return new OpenAI({ baseURL: baseURL, apiKey: apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'microsoftAzure') { + // https://learn.microsoft.com/en-us/rest/api/aifoundry/model-inference/get-chat-completions/get-chat-completions?view=rest-aifoundry-model-inference-2024-05-01-preview&tabs=HTTP + const thisConfig = settingsOfProvider[providerName] + const baseURL = `https://${thisConfig.project}.services.ai.azure.com/api/models/chat/completions??api-version=${thisConfig.azureApiVersion}` + return new OpenAI({ baseURL: baseURL, apiKey: thisConfig.apiKey, ...commonPayloadOpts }) + } + else if (providerName === 'deepseek') { const thisConfig = settingsOfProvider[providerName] return new OpenAI({ baseURL: 'https://api.deepseek.com/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts }) @@ -97,7 +131,7 @@ const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPay } -const _sendOpenAICompatibleFIM = ({ messages: { prefix, suffix, stopTokens }, onFinalMessage, onError, settingsOfProvider, modelName: modelName_, _setAborter, providerName, }: SendFIMParams_Internal) => { +const _sendOpenAICompatibleFIM = async ({ messages: { prefix, suffix, stopTokens }, onFinalMessage, onError, settingsOfProvider, modelName: modelName_, _setAborter, providerName, }: SendFIMParams_Internal) => { const { modelName, supportsFIM } = getModelCapabilities(providerName, modelName_) if (!supportsFIM) { if (modelName === modelName_) @@ -107,7 +141,7 @@ const _sendOpenAICompatibleFIM = ({ messages: { prefix, suffix, stopTokens }, on return } - const openai = newOpenAICompatibleSDK({ providerName, settingsOfProvider }) + const openai = await newOpenAICompatibleSDK({ providerName, settingsOfProvider }) openai.completions .create({ model: modelName, @@ -178,7 +212,7 @@ const openAIToolToRawToolCallObj = (name: string, toolParamsStr: string, id: str // ------------ OPENAI-COMPATIBLE ------------ -const _sendOpenAICompatibleChat = ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName: modelName_, _setAborter, providerName, chatMode, separateSystemMessage }: SendChatParams_Internal) => { +const _sendOpenAICompatibleChat = async ({ messages, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName: modelName_, _setAborter, providerName, chatMode, separateSystemMessage }: SendChatParams_Internal) => { const { modelName, specialToolFormat, @@ -199,7 +233,7 @@ const _sendOpenAICompatibleChat = ({ messages, onText, onFinalMessage, onError, : {} // instance - const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload }) + const openai: OpenAI = await newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload }) const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: modelName, messages: messages as any, @@ -300,7 +334,7 @@ const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: onError_, onError_({ error }) } try { - const openai = newOpenAICompatibleSDK({ providerName, settingsOfProvider }) + const openai = await newOpenAICompatibleSDK({ providerName, settingsOfProvider }) openai.models.list() .then(async (response) => { const models: OpenAIModel[] = [] @@ -360,7 +394,7 @@ const anthropicToolToRawToolCallObj = (toolBlock: Anthropic.Messages.ToolUseBloc } // ------------ ANTHROPIC ------------ -const sendAnthropicChat = ({ messages, providerName, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName: modelName_, _setAborter, separateSystemMessage, chatMode }: SendChatParams_Internal) => { +const sendAnthropicChat = async ({ messages, providerName, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName: modelName_, _setAborter, separateSystemMessage, chatMode }: SendChatParams_Internal) => { const { modelName, specialToolFormat, @@ -584,7 +618,7 @@ const sendOllamaFIM = ({ messages, onFinalMessage, onError, settingsOfProvider, type CallFnOfProvider = { [providerName in ProviderName]: { - sendChat: (params: SendChatParams_Internal) => void; + sendChat: (params: SendChatParams_Internal) => Promise; sendFIM: ((params: SendFIMParams_Internal) => void) | null; list: ((params: ListParams_Internal) => void) | null; } @@ -646,6 +680,27 @@ export const sendLLMMessageToProviderImplementation = { sendFIM: null, list: null, }, + + lmStudio: { + sendChat: (params) => _sendOpenAICompatibleChat(params), + sendFIM: null, // lmStudio has no suffix parameter in /completions + list: null, + }, + liteLLM: { + sendChat: (params) => _sendOpenAICompatibleChat(params), + sendFIM: null, + list: null, + }, + googleVertex: { + sendChat: (params) => _sendOpenAICompatibleChat(params), + sendFIM: null, + list: null, + }, + microsoftAzure: { + sendChat: (params) => _sendOpenAICompatibleChat(params), + sendFIM: null, + list: null, + }, } satisfies CallFnOfProvider diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts index 1554aeeb..a7c440d4 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.ts @@ -9,7 +9,7 @@ import { displayInfoOfProviderName } from '../../common/voidSettingsTypes.js'; import { sendLLMMessageToProviderImplementation } from './sendLLMMessage.impl.js'; -export const sendLLMMessage = ({ +export const sendLLMMessage = async ({ messagesType, messages: messages_, onText: onText_, @@ -108,12 +108,12 @@ export const sendLLMMessage = ({ } const { sendFIM, sendChat } = implementation if (messagesType === 'chatMessages') { - sendChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName, _setAborter, providerName, separateSystemMessage, chatMode }) + await sendChat({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName, _setAborter, providerName, separateSystemMessage, chatMode }) return } if (messagesType === 'FIMMessage') { if (sendFIM) { - sendFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName, _setAborter, providerName, separateSystemMessage }) + await sendFIM({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName, _setAborter, providerName, separateSystemMessage }) return } onError({ message: `Error: This provider does not support Autocomplete yet.`, fullError: null }) diff --git a/src/vs/workbench/contrib/void/electron-main/sendLLMMessageChannel.ts b/src/vs/workbench/contrib/void/electron-main/sendLLMMessageChannel.ts index 182ce580..b6292beb 100644 --- a/src/vs/workbench/contrib/void/electron-main/sendLLMMessageChannel.ts +++ b/src/vs/workbench/contrib/void/electron-main/sendLLMMessageChannel.ts @@ -25,7 +25,7 @@ export class LLMMessageChannel implements IServerChannel { } // aborters for above - private readonly abortRefOfRequestId: Record = {} + private readonly _infoOfRunningRequest: Record | undefined, abortRef: AbortRef }> = {} // list @@ -72,7 +72,7 @@ export class LLMMessageChannel implements IServerChannel { this._callSendLLMMessage(params) } else if (command === 'abort') { - this._callAbort(params) + await this._callAbort(params) } else if (command === 'ollamaList') { this._callOllamaList(params) @@ -93,24 +93,27 @@ export class LLMMessageChannel implements IServerChannel { private async _callSendLLMMessage(params: MainSendLLMMessageParams) { const { requestId } = params; - if (!(requestId in this.abortRefOfRequestId)) - this.abortRefOfRequestId[requestId] = { current: null } + if (!(requestId in this._infoOfRunningRequest)) + this._infoOfRunningRequest[requestId] = { waitForSend: undefined, abortRef: { current: null } } const mainThreadParams: SendLLMMessageParams = { ...params, onText: (p) => { this.llmMessageEmitters.onText.fire({ requestId, ...p }); }, onFinalMessage: (p) => { this.llmMessageEmitters.onFinalMessage.fire({ requestId, ...p }); }, onError: (p) => { console.log('sendLLM: firing err'); this.llmMessageEmitters.onError.fire({ requestId, ...p }); }, - abortRef: this.abortRefOfRequestId[requestId], + abortRef: this._infoOfRunningRequest[requestId].abortRef, } - sendLLMMessage(mainThreadParams, this.metricsService); + const p = sendLLMMessage(mainThreadParams, this.metricsService); + this._infoOfRunningRequest[requestId].waitForSend = p } - private _callAbort(params: MainLLMMessageAbortParams) { + private async _callAbort(params: MainLLMMessageAbortParams) { const { requestId } = params; - if (!(requestId in this.abortRefOfRequestId)) return - this.abortRefOfRequestId[requestId].current?.() - delete this.abortRefOfRequestId[requestId] + if (!(requestId in this._infoOfRunningRequest)) return + const { waitForSend, abortRef } = this._infoOfRunningRequest[requestId] + await waitForSend // wait for the send to finish so we know abortRef was set + abortRef?.current?.() + delete this._infoOfRunningRequest[requestId] }