mirror of
https://github.com/voideditor/void
synced 2026-05-24 09:58:23 +00:00
add openrouter claude thinking
This commit is contained in:
Andrew Pareles
parent
69697542ba
commit
1326f19a5e
4 changed files with 187 additions and 93 deletions
|
|
@ -24,13 +24,11 @@ import { VOID_CTRL_L_ACTION_ID } from '../../../actionIDs.js';
|
|||
import { VOID_OPEN_SETTINGS_ACTION_ID } from '../../../voidSettingsPane.js';
|
||||
import { ChatMode, FeatureName, isFeatureNameDisabled } from '../../../../../../../workbench/contrib/void/common/voidSettingsTypes.js';
|
||||
import { WarningBox } from '../void-settings-tsx/WarningBox.js';
|
||||
import { getModelSelectionState, getModelCapabilities } from '../../../../common/modelCapabilities.js';
|
||||
import { getModelCapabilities, getIsResoningEnabledState } from '../../../../common/modelCapabilities.js';
|
||||
import { AlertTriangle, Ban, ChevronRight, Dot, Pencil, X } from 'lucide-react';
|
||||
import { ChatMessage, StagingSelectionItem, ToolMessage, ToolRequestApproval } from '../../../../common/chatThreadServiceTypes.js';
|
||||
import { ResolveReason, ToolCallParams, ToolName, ToolNameWithApproval } from '../../../../common/toolsServiceTypes.js';
|
||||
import { ToolCallParams, ToolName, ToolNameWithApproval } from '../../../../common/toolsServiceTypes.js';
|
||||
import { JumpToFileButton, useApplyButtonHTML } from '../markdown/ApplyBlockHoverButtons.js';
|
||||
import { DiffZone } from '../../../editCodeService.js';
|
||||
import { ScrollType } from '../../../../../../../editor/common/editorCommon.js';
|
||||
import { IsRunningType } from '../../../chatThreadService.js';
|
||||
|
||||
|
||||
|
|
@ -160,11 +158,12 @@ const ReasoningOptionSlider = ({ featureName }: { featureName: FeatureName }) =>
|
|||
if (!modelSelection) return null
|
||||
|
||||
const { modelName, providerName } = modelSelection
|
||||
const { canToggleReasoning, reasoningBudgetSlider } = getModelCapabilities(providerName, modelName).supportsReasoning || {}
|
||||
const { reasoningCapabilities } = getModelCapabilities(providerName, modelName)
|
||||
const { canTurnOffReasoning, reasoningBudgetSlider } = reasoningCapabilities || {}
|
||||
|
||||
const { isReasoningEnabled } = getModelSelectionState(providerName, modelName, voidSettingsState.optionsOfModelSelection[providerName]?.[modelName])
|
||||
|
||||
if (canToggleReasoning && !reasoningBudgetSlider) { // if it's just a on/off toggle without a power slider (no models right now)
|
||||
const modelSelectionOptions = voidSettingsState.optionsOfModelSelection[providerName]?.[modelName]
|
||||
const isReasoningEnabled = getIsResoningEnabledState(providerName, modelName, modelSelectionOptions)
|
||||
if (canTurnOffReasoning && !reasoningBudgetSlider) { // if it's just a on/off toggle without a power slider (no models right now)
|
||||
return null // unused right now
|
||||
// return <div className='flex items-center gap-x-2'>
|
||||
// <span className='text-void-fg-3 text-xs pointer-events-none inline-block w-10'>{isReasoningEnabled ? 'Thinking' : 'Thinking'}</span>
|
||||
|
|
@ -183,7 +182,7 @@ const ReasoningOptionSlider = ({ featureName }: { featureName: FeatureName }) =>
|
|||
|
||||
const nSteps = 8 // only used in calculating stepSize, stepSize is what actually matters
|
||||
const stepSize = Math.round((max - min_) / nSteps)
|
||||
const min = canToggleReasoning ? min_ - stepSize : min_
|
||||
const min = canTurnOffReasoning ? min_ - stepSize : min_
|
||||
|
||||
return <div className='flex items-center gap-x-2'>
|
||||
<span className='text-void-fg-3 text-xs pointer-events-none inline-block w-10 pr-1'>Thinking</span>
|
||||
|
|
@ -195,7 +194,7 @@ const ReasoningOptionSlider = ({ featureName }: { featureName: FeatureName }) =>
|
|||
step={stepSize}
|
||||
value={value}
|
||||
onChange={(newVal) => {
|
||||
const disabled = newVal === min && canToggleReasoning
|
||||
const disabled = newVal === min && canTurnOffReasoning
|
||||
voidSettingsService.setOptionsOfModelSelection(modelSelection.providerName, modelSelection.modelName, { reasoningEnabled: !disabled, reasoningBudget: newVal })
|
||||
}}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -185,10 +185,12 @@ const VoidCommandBar = ({ uri, editor }: { uri: URI | null, editor: ICodeEditor
|
|||
|
||||
<div className="text-[var(--vscode-editor-foreground)] text-xs flex gap-4">
|
||||
<div>
|
||||
File {(currUriIdx ?? 0) + 1} of {sortedCommandBarURIs.length}
|
||||
{`File ${(currUriIdx ?? 0) + 1} of ${sortedCommandBarURIs.length}`}
|
||||
</div>
|
||||
<div>
|
||||
Diff {(currDiffIdx ?? 0) + 1} of {sortedDiffIds?.length ?? 0}
|
||||
{sortedDiffIds?.length ?? 0 === 0 ?
|
||||
'(No changes)'
|
||||
: `Diff ${(currDiffIdx ?? 0) + 1} of ${sortedDiffIds?.length ?? 0}`}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ export const defaultModelsOfProvider = {
|
|||
vLLM: [ // autodetected
|
||||
],
|
||||
openRouter: [ // https://openrouter.ai/models
|
||||
'anthropic/claude-3.7-sonnet:thinking',
|
||||
'anthropic/claude-3.7-sonnet',
|
||||
'anthropic/claude-3.5-sonnet',
|
||||
'deepseek/deepseek-r1',
|
||||
'mistralai/codestral-2501',
|
||||
|
|
@ -79,10 +81,11 @@ type ModelOptions = {
|
|||
supportsTools: false | 'anthropic-style' | 'openai-style';
|
||||
supportsFIM: boolean;
|
||||
|
||||
supportsReasoning: false | {
|
||||
reasoningCapabilities: false | {
|
||||
readonly supportsReasoning: true;
|
||||
// reasoning options if supports reasoning
|
||||
readonly canToggleReasoning: boolean; // whether or not the user can disable reasoning mode (false if the model only supports reasoning)
|
||||
readonly canIOReasoning: boolean; // whether or not the model actually outputs reasoning
|
||||
readonly canTurnOffReasoning: boolean; // whether or not the user can disable reasoning mode (false if the model only supports reasoning)
|
||||
readonly canIOReasoning: boolean; // whether or not the model actually outputs reasoning (eg o1 lets us control reasoning but not output it)
|
||||
readonly reasoningMaxOutputTokens?: number; // overrides normal maxOutputTokens // <-- UNUSED (except anthropic)
|
||||
readonly reasoningBudgetSlider?: { type: 'slider'; min: number; max: number; default: number };
|
||||
|
||||
|
|
@ -95,7 +98,7 @@ type ModelOptions = {
|
|||
|
||||
type ProviderReasoningIOSettings = {
|
||||
// include this in payload to get reasoning
|
||||
input?: { includeInPayload?: { [key: string]: any }, };
|
||||
input?: { includeInPayload?: (reasoningState: SendableReasoningInfo) => null | { [key: string]: any }, };
|
||||
// nameOfFieldInDelta: reasoning output is in response.choices[0].delta[deltaReasoningField]
|
||||
// needsManualParse: whether we must manually parse out the <think> tags
|
||||
output?:
|
||||
|
|
@ -118,7 +121,7 @@ const modelOptionsDefaults: ModelOptions = {
|
|||
supportsSystemMessage: false,
|
||||
supportsTools: false,
|
||||
supportsFIM: false,
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -127,70 +130,70 @@ const openSourceModelOptions_assumingOAICompat = {
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: false,
|
||||
supportsTools: false,
|
||||
supportsReasoning: { canToggleReasoning: false, canIOReasoning: true, openSourceThinkTags: ['<think>', '</think>'] },
|
||||
reasoningCapabilities: { supportsReasoning: true, canTurnOffReasoning: false, canIOReasoning: true, openSourceThinkTags: ['<think>', '</think>'] },
|
||||
},
|
||||
'deepseekCoderV2': {
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: false, // unstable
|
||||
supportsTools: false,
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'codestral': {
|
||||
supportsFIM: true,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
// llama
|
||||
'llama3': {
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'llama3.1': {
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'llama3.2': {
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'llama3.3': {
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
// qwen
|
||||
'qwen2.5coder': {
|
||||
supportsFIM: true,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'qwq': {
|
||||
supportsFIM: false, // no FIM, yes reasoning
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: { canToggleReasoning: false, canIOReasoning: true, openSourceThinkTags: ['<think>', '</think>'] },
|
||||
reasoningCapabilities: { supportsReasoning: true, canTurnOffReasoning: false, canIOReasoning: true, openSourceThinkTags: ['<think>', '</think>'] },
|
||||
},
|
||||
// FIM only
|
||||
'starcoder2': {
|
||||
supportsFIM: true,
|
||||
supportsSystemMessage: false,
|
||||
supportsTools: false,
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'codegemma:2b': {
|
||||
supportsFIM: true,
|
||||
supportsSystemMessage: false,
|
||||
supportsTools: false,
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
} as const satisfies { [s: string]: Partial<ModelOptions> }
|
||||
|
||||
|
|
@ -233,8 +236,9 @@ const anthropicModelOptions = {
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: {
|
||||
canToggleReasoning: true,
|
||||
reasoningCapabilities: {
|
||||
supportsReasoning: true,
|
||||
canTurnOffReasoning: true,
|
||||
canIOReasoning: true,
|
||||
reasoningMaxOutputTokens: 64_000, // can bump it to 128_000 with beta mode output-128k-2025-02-19
|
||||
reasoningBudgetSlider: { type: 'slider', min: 1024, max: 32_000, default: 1024 }, // they recommend batching if max > 32_000
|
||||
|
|
@ -247,7 +251,7 @@ const anthropicModelOptions = {
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'claude-3-5-haiku-20241022': {
|
||||
contextWindow: 200_000,
|
||||
|
|
@ -256,7 +260,7 @@ const anthropicModelOptions = {
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'claude-3-opus-20240229': {
|
||||
contextWindow: 200_000,
|
||||
|
|
@ -265,7 +269,7 @@ const anthropicModelOptions = {
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'claude-3-sonnet-20240229': { // no point of using this, but including this for people who put it in
|
||||
contextWindow: 200_000, cost: { input: 3.00, output: 15.00 },
|
||||
|
|
@ -273,11 +277,21 @@ const anthropicModelOptions = {
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'separated',
|
||||
supportsTools: 'anthropic-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
}
|
||||
} as const satisfies { [s: string]: ModelOptions }
|
||||
|
||||
const anthropicSettings: ProviderSettings = {
|
||||
providerReasoningIOSettings: {
|
||||
input: {
|
||||
includeInPayload: (reasoningInfo) => {
|
||||
if (reasoningInfo?.type === 'budgetEnabled') {
|
||||
return { thinking: { type: 'enabled', budget_tokens: reasoningInfo.reasoningBudget } }
|
||||
}
|
||||
return null
|
||||
}
|
||||
},
|
||||
},
|
||||
modelOptions: anthropicModelOptions,
|
||||
modelOptionsFallback: (modelName) => {
|
||||
let fallbackName: keyof typeof anthropicModelOptions | null = null
|
||||
|
|
@ -288,7 +302,7 @@ const anthropicSettings: ProviderSettings = {
|
|||
if (modelName.includes('claude-3-sonnet')) fallbackName = 'claude-3-sonnet-20240229'
|
||||
if (fallbackName) return { modelName: fallbackName, ...anthropicModelOptions[fallbackName] }
|
||||
return { modelName, ...modelOptionsDefaults, maxOutputTokens: 4_096 }
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -301,7 +315,7 @@ const openAIModelOptions = { // https://platform.openai.com/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsTools: false,
|
||||
supportsSystemMessage: 'developer-role',
|
||||
supportsReasoning: { canIOReasoning: false, canToggleReasoning: false }, // it doesn't actually output reasoning, but our logic is fine with it
|
||||
reasoningCapabilities: { supportsReasoning: true, canIOReasoning: false, canTurnOffReasoning: false }, // it doesn't actually output reasoning, but our logic is fine with it
|
||||
},
|
||||
'o3-mini': {
|
||||
contextWindow: 200_000,
|
||||
|
|
@ -310,7 +324,7 @@ const openAIModelOptions = { // https://platform.openai.com/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsTools: false,
|
||||
supportsSystemMessage: 'developer-role',
|
||||
supportsReasoning: { canIOReasoning: false, canToggleReasoning: false },
|
||||
reasoningCapabilities: { supportsReasoning: true, canIOReasoning: false, canTurnOffReasoning: false },
|
||||
},
|
||||
'gpt-4o': {
|
||||
contextWindow: 128_000,
|
||||
|
|
@ -319,7 +333,7 @@ const openAIModelOptions = { // https://platform.openai.com/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsTools: 'openai-style',
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'o1-mini': {
|
||||
contextWindow: 128_000,
|
||||
|
|
@ -328,7 +342,7 @@ const openAIModelOptions = { // https://platform.openai.com/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsTools: false,
|
||||
supportsSystemMessage: false, // does not support any system
|
||||
supportsReasoning: { canIOReasoning: false, canToggleReasoning: false },
|
||||
reasoningCapabilities: { supportsReasoning: true, canIOReasoning: false, canTurnOffReasoning: false },
|
||||
},
|
||||
'gpt-4o-mini': {
|
||||
contextWindow: 128_000,
|
||||
|
|
@ -337,7 +351,7 @@ const openAIModelOptions = { // https://platform.openai.com/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsTools: 'openai-style',
|
||||
supportsSystemMessage: 'system-role', // ??
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
} as const satisfies { [s: string]: ModelOptions }
|
||||
|
||||
|
|
@ -363,7 +377,7 @@ const xAIModelOptions = {
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
} as const satisfies { [s: string]: ModelOptions }
|
||||
|
||||
|
|
@ -387,7 +401,7 @@ const geminiModelOptions = { // https://ai.google.dev/gemini-api/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style', // we are assuming OpenAI SDK when calling gemini
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'gemini-2.0-flash-lite-preview-02-05': {
|
||||
contextWindow: 1_048_576,
|
||||
|
|
@ -396,7 +410,7 @@ const geminiModelOptions = { // https://ai.google.dev/gemini-api/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'gemini-1.5-flash': {
|
||||
contextWindow: 1_048_576,
|
||||
|
|
@ -405,7 +419,7 @@ const geminiModelOptions = { // https://ai.google.dev/gemini-api/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'gemini-1.5-pro': {
|
||||
contextWindow: 2_097_152,
|
||||
|
|
@ -414,7 +428,7 @@ const geminiModelOptions = { // https://ai.google.dev/gemini-api/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'gemini-1.5-flash-8b': {
|
||||
contextWindow: 1_048_576,
|
||||
|
|
@ -423,7 +437,7 @@ const geminiModelOptions = { // https://ai.google.dev/gemini-api/docs/pricing
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
} as const satisfies { [s: string]: ModelOptions }
|
||||
|
||||
|
|
@ -469,7 +483,7 @@ const groqModelOptions = { // https://console.groq.com/docs/models, https://groq
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'llama-3.1-8b-instant': {
|
||||
contextWindow: 128_000,
|
||||
|
|
@ -478,7 +492,7 @@ const groqModelOptions = { // https://console.groq.com/docs/models, https://groq
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'qwen-2.5-coder-32b': {
|
||||
contextWindow: 128_000,
|
||||
|
|
@ -487,7 +501,7 @@ const groqModelOptions = { // https://console.groq.com/docs/models, https://groq
|
|||
supportsFIM: false, // unfortunately looks like no FIM support on groq
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'qwen-qwq-32b': { // https://huggingface.co/Qwen/QwQ-32B
|
||||
contextWindow: 128_000,
|
||||
|
|
@ -496,11 +510,21 @@ const groqModelOptions = { // https://console.groq.com/docs/models, https://groq
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: { canIOReasoning: true, canToggleReasoning: false, openSourceThinkTags: ['<think>', '</think>'] }, // we're using reasoning_format:parsed so really don't need to know openSourceThinkTags
|
||||
reasoningCapabilities: { supportsReasoning: true, canIOReasoning: true, canTurnOffReasoning: false, openSourceThinkTags: ['<think>', '</think>'] }, // we're using reasoning_format:parsed so really don't need to know openSourceThinkTags
|
||||
},
|
||||
} as const satisfies { [s: string]: ModelOptions }
|
||||
const groqSettings: ProviderSettings = {
|
||||
providerReasoningIOSettings: { input: { includeInPayload: { reasoning_format: 'parsed' } }, output: { nameOfFieldInDelta: 'reasoning' }, }, // Must be set to either parsed or hidden when using tool calling https://console.groq.com/docs/reasoning
|
||||
providerReasoningIOSettings: {
|
||||
input: {
|
||||
includeInPayload: (reasoningInfo) => {
|
||||
if (reasoningInfo?.type === 'budgetEnabled') {
|
||||
return { reasoning_format: 'parsed' }
|
||||
}
|
||||
return null
|
||||
}
|
||||
},
|
||||
output: { nameOfFieldInDelta: 'reasoning' },
|
||||
}, // Must be set to either parsed or hidden when using tool calling https://console.groq.com/docs/reasoning
|
||||
modelOptions: groqModelOptions,
|
||||
modelOptionsFallback: (modelName) => { return null }
|
||||
}
|
||||
|
|
@ -536,6 +560,21 @@ const openRouterModelOptions_assumingOpenAICompat = {
|
|||
maxOutputTokens: null,
|
||||
cost: { input: 0.8, output: 2.4 },
|
||||
},
|
||||
'anthropic/claude-3.7-sonnet:thinking': {
|
||||
contextWindow: 200_000,
|
||||
maxOutputTokens: null,
|
||||
cost: { input: 3.00, output: 15.00 },
|
||||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
reasoningCapabilities: { // same as anthropic, see above
|
||||
supportsReasoning: true,
|
||||
canTurnOffReasoning: false,
|
||||
canIOReasoning: true,
|
||||
reasoningMaxOutputTokens: 64_000,
|
||||
reasoningBudgetSlider: { type: 'slider', min: 1024, max: 32_000, default: 1024 }, // they recommend batching if max > 32_000
|
||||
},
|
||||
},
|
||||
'anthropic/claude-3.7-sonnet': {
|
||||
contextWindow: 200_000,
|
||||
maxOutputTokens: null,
|
||||
|
|
@ -543,7 +582,7 @@ const openRouterModelOptions_assumingOpenAICompat = {
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: { canIOReasoning: true, canToggleReasoning: false }, // TODO!!! false for now
|
||||
reasoningCapabilities: false, // stupidly, openrouter separates thinking from non-thinking
|
||||
},
|
||||
'anthropic/claude-3.5-sonnet': {
|
||||
contextWindow: 200_000,
|
||||
|
|
@ -552,7 +591,7 @@ const openRouterModelOptions_assumingOpenAICompat = {
|
|||
supportsFIM: false,
|
||||
supportsSystemMessage: 'system-role',
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'mistralai/codestral-2501': {
|
||||
...openSourceModelOptions_assumingOAICompat.codestral,
|
||||
|
|
@ -560,7 +599,7 @@ const openRouterModelOptions_assumingOpenAICompat = {
|
|||
maxOutputTokens: null,
|
||||
cost: { input: 0.3, output: 0.9 },
|
||||
supportsTools: 'openai-style',
|
||||
supportsReasoning: false,
|
||||
reasoningCapabilities: false,
|
||||
},
|
||||
'qwen/qwen-2.5-coder-32b-instruct': {
|
||||
...openSourceModelOptions_assumingOAICompat['qwen2.5coder'],
|
||||
|
|
@ -581,7 +620,18 @@ const openRouterModelOptions_assumingOpenAICompat = {
|
|||
const openRouterSettings: ProviderSettings = {
|
||||
// reasoning: OAICompat + response.choices[0].delta.reasoning : payload should have {include_reasoning: true} https://openrouter.ai/announcements/reasoning-tokens-for-thinking-models
|
||||
providerReasoningIOSettings: {
|
||||
input: { includeInPayload: { include_reasoning: true } },
|
||||
input: {
|
||||
includeInPayload: (reasoningInfo) => {
|
||||
if (reasoningInfo?.type === 'budgetEnabled') {
|
||||
return {
|
||||
reasoning: {
|
||||
max_tokens: reasoningInfo.reasoningBudget
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
},
|
||||
output: { nameOfFieldInDelta: 'reasoning' },
|
||||
},
|
||||
modelOptions: openRouterModelOptions_assumingOpenAICompat,
|
||||
|
|
@ -632,12 +682,45 @@ export const getProviderCapabilities = (providerName: ProviderName) => {
|
|||
return { providerReasoningIOSettings }
|
||||
}
|
||||
|
||||
// state from optionsOfModelSelection
|
||||
export const getModelSelectionState = (providerName: ProviderName, modelName: string, modelSelectionOptions: ModelSelectionOptions | undefined): { isReasoningEnabled: boolean, reasoningBudget: number | undefined } => {
|
||||
const { canToggleReasoning, reasoningBudgetSlider } = getModelCapabilities(providerName, modelName).supportsReasoning || {}
|
||||
|
||||
const defaultEnabledVal = canToggleReasoning ? true : false
|
||||
export type SendableReasoningInfo = {
|
||||
type: 'budgetEnabled',
|
||||
isReasoningEnabled: true,
|
||||
reasoningBudget: number,
|
||||
} | null
|
||||
|
||||
|
||||
|
||||
export const getIsResoningEnabledState = (
|
||||
providerName: ProviderName,
|
||||
modelName: string,
|
||||
modelSelectionOptions: ModelSelectionOptions | undefined,
|
||||
) => {
|
||||
const { supportsReasoning } = getModelCapabilities(providerName, modelName).reasoningCapabilities || {}
|
||||
if (!supportsReasoning) return false
|
||||
|
||||
const defaultEnabledVal = true // if can't toggle reasoning, then this must be true. just true as default
|
||||
const isReasoningEnabled = modelSelectionOptions?.reasoningEnabled ?? defaultEnabledVal
|
||||
const reasoningBudget = reasoningBudgetSlider?.type === 'slider' ? modelSelectionOptions?.reasoningBudget ?? reasoningBudgetSlider?.default : undefined
|
||||
return { isReasoningEnabled, reasoningBudget }
|
||||
return isReasoningEnabled
|
||||
}
|
||||
|
||||
|
||||
// used to force reasoning state (complex) into something simple we can just read from when sending a message
|
||||
export const getSendableReasoningInfo = (
|
||||
providerName: ProviderName,
|
||||
modelName: string,
|
||||
modelSelectionOptions: ModelSelectionOptions | undefined,
|
||||
): SendableReasoningInfo => {
|
||||
|
||||
const { canIOReasoning, reasoningBudgetSlider } = getModelCapabilities(providerName, modelName).reasoningCapabilities || {}
|
||||
if (!canIOReasoning) return null
|
||||
const isReasoningEnabled = getIsResoningEnabledState(providerName, modelName, modelSelectionOptions)
|
||||
if (!isReasoningEnabled) return null
|
||||
|
||||
// check for reasoning budget
|
||||
const reasoningBudget = reasoningBudgetSlider?.type === 'slider' ? modelSelectionOptions?.reasoningBudget ?? reasoningBudgetSlider?.default : undefined
|
||||
if (reasoningBudget) {
|
||||
return { type: 'budgetEnabled', isReasoningEnabled: isReasoningEnabled, reasoningBudget: reasoningBudget }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { extractReasoningOnFinalMessage, extractReasoningOnTextWrapper } from '.
|
|||
import { LLMChatMessage, LLMFIMMessage, ModelListParams, OllamaModelResponse, OnError, OnFinalMessage, OnText } from '../../common/sendLLMMessageTypes.js';
|
||||
import { defaultProviderSettings, displayInfoOfProviderName, ModelSelectionOptions, ProviderName, SettingsOfProvider } from '../../common/voidSettingsTypes.js';
|
||||
import { prepareFIMMessage, prepareMessages } from './preprocessLLMMessages.js';
|
||||
import { getModelSelectionState, getModelCapabilities, getProviderCapabilities } from '../../common/modelCapabilities.js';
|
||||
import { getSendableReasoningInfo, getModelCapabilities, getProviderCapabilities } from '../../common/modelCapabilities.js';
|
||||
import { InternalToolInfo, ToolName, isAToolName } from '../../common/toolsServiceTypes.js';
|
||||
|
||||
|
||||
|
|
@ -150,32 +150,41 @@ const _sendOpenAICompatibleFIM = ({ messages: messages_, onFinalMessage, onError
|
|||
|
||||
|
||||
|
||||
const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelName: modelName_, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => {
|
||||
const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinalMessage, onError, settingsOfProvider, modelSelectionOptions, modelName: modelName_, _setAborter, providerName, aiInstructions, tools: tools_ }: SendChatParams_Internal) => {
|
||||
const {
|
||||
modelName,
|
||||
supportsReasoning,
|
||||
supportsSystemMessage,
|
||||
supportsTools,
|
||||
// maxOutputTokens, right now we are ignoring this
|
||||
// maxOutputTokens,
|
||||
reasoningCapabilities,
|
||||
} = getModelCapabilities(providerName, modelName_)
|
||||
|
||||
const {
|
||||
canIOReasoning,
|
||||
openSourceThinkTags,
|
||||
} = supportsReasoning || {}
|
||||
|
||||
|
||||
const { providerReasoningIOSettings } = getProviderCapabilities(providerName)
|
||||
|
||||
const { messages } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, supportsAnthropicReasoningSignature: false })
|
||||
// reasoning
|
||||
const { canIOReasoning, openSourceThinkTags, } = reasoningCapabilities || {}
|
||||
const reasoningInfo = getSendableReasoningInfo(providerName, modelName_, modelSelectionOptions) // user's modelName_ here
|
||||
const includeInPayload = providerReasoningIOSettings?.input?.includeInPayload?.(reasoningInfo) || {}
|
||||
|
||||
// tools
|
||||
const tools = (supportsTools && ((tools_?.length ?? 0) !== 0)) ? tools_?.map(tool => toOpenAICompatibleTool(tool)) : undefined
|
||||
|
||||
const includeInPayload = canIOReasoning ? providerReasoningIOSettings?.input?.includeInPayload || {} : {}
|
||||
|
||||
const toolsObj = tools ? { tools: tools, tool_choice: 'auto', parallel_tool_calls: false, } as const : {}
|
||||
const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload })
|
||||
const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { model: modelName, messages: messages, stream: true, ...toolsObj, }
|
||||
|
||||
// max tokens
|
||||
// const maxTokens = reasoningInfo?.isReasoningEnabled && reasoningCapabilities ? reasoningCapabilities.reasoningMaxOutputTokens : maxOutputTokens
|
||||
|
||||
// instance
|
||||
const { messages } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, supportsAnthropicReasoningSignature: false })
|
||||
const openai: OpenAI = newOpenAICompatibleSDK({ providerName, settingsOfProvider, includeInPayload })
|
||||
const options: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
|
||||
model: modelName,
|
||||
messages: messages,
|
||||
stream: true,
|
||||
// max_completion_tokens: maxTokens,
|
||||
...toolsObj,
|
||||
}
|
||||
|
||||
// open source models - manually parse think tokens
|
||||
const { needsManualParse: needsManualReasoningParse, nameOfFieldInDelta: nameOfReasoningFieldInDelta } = providerReasoningIOSettings?.output ?? {}
|
||||
const manuallyParseReasoning = needsManualReasoningParse && canIOReasoning && openSourceThinkTags
|
||||
if (manuallyParseReasoning) {
|
||||
|
|
@ -300,31 +309,32 @@ const sendAnthropicChat = ({ messages: messages_, providerName, onText, onFinalM
|
|||
supportsSystemMessage,
|
||||
supportsTools,
|
||||
maxOutputTokens,
|
||||
supportsReasoning,
|
||||
reasoningCapabilities,
|
||||
} = getModelCapabilities(providerName, modelName_)
|
||||
const {
|
||||
isReasoningEnabled,
|
||||
reasoningBudget,
|
||||
} = getModelSelectionState(providerName, modelName_, modelSelectionOptions) // user's modelName_ here
|
||||
|
||||
const { messages, separateSystemMessageStr } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, supportsAnthropicReasoningSignature: true })
|
||||
|
||||
const thisConfig = settingsOfProvider.anthropic
|
||||
const anthropic = new Anthropic({ apiKey: thisConfig.apiKey, dangerouslyAllowBrowser: true });
|
||||
const { providerReasoningIOSettings } = getProviderCapabilities(providerName)
|
||||
|
||||
// reasoning
|
||||
const reasoningInfo = getSendableReasoningInfo(providerName, modelName_, modelSelectionOptions) // user's modelName_ here
|
||||
const includeInPayload = providerReasoningIOSettings?.input?.includeInPayload?.(reasoningInfo) || {}
|
||||
|
||||
// tools
|
||||
const tools = ((tools_?.length ?? 0) !== 0) ? tools_?.map(tool => toAnthropicTool(tool)) : undefined
|
||||
|
||||
|
||||
const toolsObj: Partial<Anthropic.Messages.MessageStreamParams> = tools ? {
|
||||
tools: tools,
|
||||
tool_choice: { type: 'auto', disable_parallel_tool_use: true } // one tool at a time
|
||||
} : {}
|
||||
|
||||
// anthropic-specific - max tokens
|
||||
const maxTokens = reasoningInfo?.isReasoningEnabled && reasoningCapabilities ? reasoningCapabilities.reasoningMaxOutputTokens : maxOutputTokens
|
||||
|
||||
const enableThinking = supportsReasoning && isReasoningEnabled && reasoningBudget
|
||||
const maxTokens = enableThinking ? supportsReasoning.reasoningMaxOutputTokens : maxOutputTokens
|
||||
const thinkingObj: Partial<Anthropic.Messages.MessageStreamParams> = enableThinking ? {
|
||||
thinking: { type: 'enabled', budget_tokens: reasoningBudget } // thinking enabled
|
||||
} : {}
|
||||
// instance
|
||||
const { messages, separateSystemMessageStr } = prepareMessages({ messages: messages_, aiInstructions, supportsSystemMessage, supportsTools, supportsAnthropicReasoningSignature: true })
|
||||
const anthropic = new Anthropic({
|
||||
apiKey: thisConfig.apiKey,
|
||||
dangerouslyAllowBrowser: true
|
||||
});
|
||||
|
||||
const stream = anthropic.messages.stream({
|
||||
system: separateSystemMessageStr,
|
||||
|
|
@ -332,7 +342,7 @@ const sendAnthropicChat = ({ messages: messages_, providerName, onText, onFinalM
|
|||
model: modelName,
|
||||
max_tokens: maxTokens ?? 4_096, // anthropic requires this
|
||||
...toolsObj,
|
||||
...thinkingObj,
|
||||
...includeInPayload,
|
||||
})
|
||||
|
||||
// when receive text
|
||||
|
|
|
|||
Loading…
Reference in a new issue