mirror of
https://github.com/voideditor/void
synced 2026-05-24 01:48:25 +00:00
Merge pull request #321 from jcommaret/feat_mistral-199
[Provider]Mistral -😻- Mistral working : [OpenAICompatibleModel Chat + tools + sendMistralFIM]
This commit is contained in:
Andrew Pareles
GitHub
commit
4d02381dab
3 changed files with 131 additions and 26 deletions
|
|
@ -39,8 +39,11 @@ export const defaultProviderSettings = {
|
|||
apiKey: '',
|
||||
},
|
||||
xAI: {
|
||||
apiKey: ''
|
||||
apiKey: '',
|
||||
},
|
||||
mistral: {
|
||||
apiKey: '',
|
||||
}
|
||||
} as const
|
||||
|
||||
|
||||
|
|
@ -98,13 +101,12 @@ export const defaultModelsOfProvider = {
|
|||
'llama-3.1-8b-instant',
|
||||
// 'qwen-2.5-coder-32b', // preview mode (experimental)
|
||||
],
|
||||
// not supporting mistral right now- it's last on Void usage, and a huge pain to set up since it's nonstandard (it supports codestral FIM but it's on v1/fim/completions, etc)
|
||||
// mistral: [ // https://docs.mistral.ai/getting-started/models/models_overview/
|
||||
// 'codestral-latest',
|
||||
// 'mistral-large-latest',
|
||||
// 'ministral-3b-latest',
|
||||
// 'ministral-8b-latest',
|
||||
// ],
|
||||
mistral: [ // https://docs.mistral.ai/getting-started/models/models_overview/
|
||||
'codestral-latest',
|
||||
'mistral-large-latest',
|
||||
'ministral-3b-latest',
|
||||
'ministral-8b-latest',
|
||||
],
|
||||
openAICompatible: [], // fallback
|
||||
} as const satisfies Record<ProviderName, string[]>
|
||||
|
||||
|
|
@ -170,12 +172,9 @@ const modelOptionsDefaults: VoidStaticModelInfo = {
|
|||
reasoningCapabilities: false,
|
||||
}
|
||||
|
||||
|
||||
|
||||
// TODO!!! double check all context sizes below
|
||||
// TODO!!! add openrouter common models
|
||||
// TODO!!! allow user to modify capabilities and tell them if autodetected model or falling back
|
||||
|
||||
const openSourceModelOptions_assumingOAICompat = {
|
||||
'deepseekR1': {
|
||||
supportsFIM: false,
|
||||
|
|
@ -624,6 +623,55 @@ const deepseekSettings: VoidStaticProviderInfo = {
|
|||
modelOptionsFallback: (modelName) => { return null }
|
||||
}
|
||||
|
||||
|
||||
|
||||
// ---------------- MISTRAL ----------------
|
||||
|
||||
const mistralModelOptions = { // https://mistral.ai/products/la-plateforme#pricing https://docs.mistral.ai/getting-started/models/models_overview/#premier-models
|
||||
'mistral-large-latest': {
|
||||
contextWindow: 131_000,
|
||||
maxOutputTokens: 8_192,
|
||||
cost: { input: 2.00, output: 6.00 },
|
||||
supportsFIM: false,
|
||||
downloadable: { sizeGb: 73 },
|
||||
supportsSystemMessage: 'system-role',
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'codestral-latest': {
|
||||
contextWindow: 256_000,
|
||||
maxOutputTokens: 8_192,
|
||||
cost: { input: 0.30, output: 0.90 },
|
||||
supportsFIM: true,
|
||||
downloadable: { sizeGb: 13 },
|
||||
supportsSystemMessage: 'system-role',
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'ministral-8b-latest': { // ollama 'mistral'
|
||||
contextWindow: 131_000,
|
||||
maxOutputTokens: 4_096,
|
||||
cost: { input: 0.10, output: 0.10 },
|
||||
supportsFIM: false,
|
||||
downloadable: { sizeGb: 4.1 },
|
||||
supportsSystemMessage: 'system-role',
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'ministral-3b-latest': {
|
||||
contextWindow: 131_000,
|
||||
maxOutputTokens: 4_096,
|
||||
cost: { input: 0.04, output: 0.04 },
|
||||
supportsFIM: false,
|
||||
downloadable: { sizeGb: 'not-known' },
|
||||
supportsSystemMessage: 'system-role',
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
} as const satisfies { [s: string]: VoidStaticModelInfo }
|
||||
|
||||
const mistralSettings: VoidStaticProviderInfo = {
|
||||
modelOptions: mistralModelOptions,
|
||||
modelOptionsFallback: (modelName) => { return null },
|
||||
}
|
||||
|
||||
|
||||
// ---------------- GROQ ----------------
|
||||
const groqModelOptions = { // https://console.groq.com/docs/models, https://groq.com/pricing/
|
||||
'llama-3.3-70b-versatile': {
|
||||
|
|
@ -890,8 +938,7 @@ const modelSettingsOfProvider: { [providerName in ProviderName]: VoidStaticProvi
|
|||
vLLM: vLLMSettings,
|
||||
ollama: ollamaSettings,
|
||||
openAICompatible: openaiCompatible,
|
||||
|
||||
// TODO!!!
|
||||
mistral: mistralSettings,
|
||||
// googleVertex: {},
|
||||
// microsoftAzure: {},
|
||||
// openHands: {},
|
||||
|
|
|
|||
|
|
@ -108,6 +108,11 @@ export const displayInfoOfProviderName = (providerName: ProviderName): DisplayIn
|
|||
title: 'Grok (xAI)',
|
||||
}
|
||||
}
|
||||
else if (providerName === 'mistral') {
|
||||
return {
|
||||
title: 'Mistral API',
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
throw new Error(`descOfProviderName: Unknown provider name: "${providerName}"`)
|
||||
|
|
@ -134,7 +139,8 @@ export const displayInfoOfSettingName = (providerName: ProviderName, settingName
|
|||
providerName === 'groq' ? 'gsk_key...' :
|
||||
providerName === 'openAICompatible' ? 'sk-key...' :
|
||||
providerName === 'xAI' ? 'xai-key...' :
|
||||
'',
|
||||
providerName === 'mistral' ? 'api-key...' :
|
||||
'',
|
||||
|
||||
subTextMd: providerName === 'anthropic' ? 'Get your [API Key here](https://console.anthropic.com/settings/keys).' :
|
||||
providerName === 'openAI' ? 'Get your [API Key here](https://platform.openai.com/api-keys).' :
|
||||
|
|
@ -143,8 +149,9 @@ export const displayInfoOfSettingName = (providerName: ProviderName, settingName
|
|||
providerName === 'gemini' ? 'Get your [API Key here](https://aistudio.google.com/apikey).' :
|
||||
providerName === 'groq' ? 'Get your [API Key here](https://console.groq.com/keys).' :
|
||||
providerName === 'xAI' ? 'Get your [API Key here](https://console.x.ai).' :
|
||||
providerName === 'openAICompatible' ? undefined :
|
||||
'',
|
||||
providerName === 'mistral' ? 'Get your [API Key here](https://console.mistral.ai/api-keys).' :
|
||||
providerName === 'openAICompatible' ? undefined :
|
||||
'',
|
||||
isPasswordField: true,
|
||||
}
|
||||
}
|
||||
|
|
@ -234,31 +241,37 @@ export const defaultSettingsOfProvider: SettingsOfProvider = {
|
|||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.xAI),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
groq: { // aggregator
|
||||
mistral: {
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.mistral,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.mistral),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
groq: { // aggregator (serves models from multiple providers)
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.groq,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.groq),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
openRouter: { // aggregator
|
||||
openRouter: { // aggregator (serves models from multiple providers)
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.openRouter,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.openRouter),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
openAICompatible: { // aggregator
|
||||
openAICompatible: { // aggregator (serves models from multiple providers)
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.openAICompatible,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.openAICompatible),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
ollama: { // aggregator
|
||||
ollama: { // aggregator (serves models from multiple providers)
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.ollama,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.ollama),
|
||||
_didFillInProviderSettings: undefined,
|
||||
},
|
||||
vLLM: { // aggregator
|
||||
vLLM: { // aggregator (serves models from multiple providers)
|
||||
...defaultCustomSettings,
|
||||
...defaultProviderSettings.vLLM,
|
||||
...modelInfoOfDefaultModelNames(defaultModelsOfProvider.vLLM),
|
||||
|
|
|
|||
|
|
@ -6,6 +6,9 @@
|
|||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import { Ollama } from 'ollama';
|
||||
import OpenAI, { ClientOptions } from 'openai';
|
||||
import { MistralCore } from '@mistralai/mistralai/core.js';
|
||||
import { fimComplete } from '@mistralai/mistralai/funcs/fimComplete.js';
|
||||
|
||||
|
||||
import { LLMChatMessage, LLMFIMMessage, ModelListParams, OllamaModelResponse, OnError, OnFinalMessage, OnText } from '../../common/sendLLMMessageTypes.js';
|
||||
import { ChatMode, displayInfoOfProviderName, ModelSelectionOptions, ProviderName, SettingsOfProvider } from '../../common/voidSettingsTypes.js';
|
||||
|
|
@ -84,6 +87,10 @@ const newOpenAICompatibleSDK = ({ settingsOfProvider, providerName, includeInPay
|
|||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.x.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
else if (providerName === 'mistral') {
|
||||
const thisConfig = settingsOfProvider[providerName]
|
||||
return new OpenAI({ baseURL: 'https://api.mistral.ai/v1', apiKey: thisConfig.apiKey, ...commonPayloadOpts })
|
||||
}
|
||||
|
||||
else throw new Error(`Void providerName was invalid: ${providerName}.`)
|
||||
}
|
||||
|
|
@ -349,6 +356,44 @@ const sendAnthropicChat = ({ messages: messages_, providerName, onText, onFinalM
|
|||
_setAborter(() => stream.controller.abort())
|
||||
}
|
||||
|
||||
|
||||
|
||||
// ------------ MISTRAL ------------
|
||||
// https://docs.mistral.ai/api/#tag/fim
|
||||
const sendMistralFIM = ({ messages: messages_, onFinalMessage, onError, settingsOfProvider, modelName: modelName_, _setAborter, providerName, aiInstructions, modelSelectionOptions }: SendFIMParams_Internal) => {
|
||||
const { modelName, supportsFIM } = getModelCapabilities(providerName, modelName_)
|
||||
if (!supportsFIM) {
|
||||
if (modelName === modelName_)
|
||||
onError({ message: `Model ${modelName} does not support FIM.`, fullError: null })
|
||||
else
|
||||
onError({ message: `Model ${modelName_} (${modelName}) does not support FIM.`, fullError: null })
|
||||
return
|
||||
}
|
||||
const messages = prepareFIMMessage({ messages: messages_, aiInstructions })
|
||||
|
||||
const mistral = new MistralCore({ apiKey: settingsOfProvider.mistral.apiKey })
|
||||
fimComplete(mistral,
|
||||
{
|
||||
model: modelName,
|
||||
prompt: messages.prefix,
|
||||
suffix: messages.suffix,
|
||||
stream: false,
|
||||
maxTokens: messages.maxTokens,
|
||||
stop: messages.stopTokens,
|
||||
})
|
||||
.then(async response => {
|
||||
let content = response?.ok ? response.value.choices?.[0]?.message?.content ?? '' : '';
|
||||
const fullText = typeof content === 'string' ? content
|
||||
: content.map(chunk => (chunk.type === 'text' ? chunk.text : '')).join('')
|
||||
|
||||
onFinalMessage({ fullText, fullReasoning: '', anthropicReasoning: null });
|
||||
})
|
||||
.catch(error => {
|
||||
onError({ message: error + '', fullError: error });
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
// ------------ OLLAMA ------------
|
||||
const newOllamaSDK = ({ endpoint }: { endpoint: string }) => {
|
||||
// if endpoint is empty, normally ollama will send to 11434, but we want it to fail - the user should type it in
|
||||
|
|
@ -445,11 +490,11 @@ export const sendLLMMessageToProviderImplementation = {
|
|||
sendFIM: null,
|
||||
list: null,
|
||||
},
|
||||
// mistral: {
|
||||
// sendChat: , // TODO
|
||||
// sendFIM: , // TODO // https://docs.mistral.ai/api/#tag/fim
|
||||
// list: null,
|
||||
// },
|
||||
mistral: {
|
||||
sendChat: (params) => _sendOpenAICompatibleChat(params),
|
||||
sendFIM: (params) => sendMistralFIM(params),
|
||||
list: null,
|
||||
},
|
||||
ollama: {
|
||||
sendChat: (params) => _sendOpenAICompatibleChat(params),
|
||||
sendFIM: sendOllamaFIM,
|
||||
|
|
|
|||
Loading…
Reference in a new issue