autodetect language

This commit is contained in:
mp 2024-12-23 16:21:33 -08:00
parent ee8b98ff74
commit f08fd04af5
7 changed files with 75 additions and 62 deletions

View file

@ -385,7 +385,7 @@ type FeatureFlagDisplayInfo = {
export const displayInfoOfFeatureFlag = (featureFlag: FeatureFlagName): FeatureFlagDisplayInfo => {
if (featureFlag === 'autoRefreshModels') {
return {
description: `Automatically scan for and enable local models.`, // ${`refreshableProviderNames.map(providerName => titleOfProviderName(providerName)).join(', ')`}
description: `Automatically scan for and enable local providers like Ollama.`, // ${`refreshableProviderNames.map(providerName => titleOfProviderName(providerName)).join(', ')`}
}
}
throw new Error(`featureFlagInfo: Unknown feature flag: "${featureFlag}"`)

View file

@ -5,68 +5,65 @@
import React, { ReactNode } from "react"
import { VoidCodeEditor } from '../util/inputs.js';
import { ILanguageService } from '../../../../../../../editor/common/languages/language.js';
const extensionMap: { [key: string]: string } = {
// Web
'html': 'html',
'htm': 'html',
'css': 'css',
'scss': 'scss',
'less': 'less',
'js': 'javascript',
'jsx': 'javascript',
'ts': 'typescript',
'tsx': 'typescript',
'json': 'json',
'jsonc': 'json',
// Programming Languages
'py': 'python',
'java': 'java',
'cpp': 'cpp',
'cc': 'cpp',
'h': 'cpp',
'hpp': 'cpp',
'cs': 'csharp',
'go': 'go',
'rs': 'rust',
'rb': 'ruby',
'php': 'php',
'sh': 'shell',
'bash': 'shell',
'zsh': 'shell',
// Markup/Config
'md': 'markdown',
'markdown': 'markdown',
'xml': 'xml',
'svg': 'xml',
'yaml': 'yaml',
'yml': 'yaml',
'ini': 'ini',
'toml': 'ini',
// Other
'sql': 'sql',
'graphql': 'graphql',
'gql': 'graphql',
'dockerfile': 'dockerfile',
'docker': 'dockerfile'
};
export function getLanguageFromFileName(fileName: string): string {
if (!fileName) return 'plaintext';
const ext = fileName.toLowerCase().split('.').pop();
if (!ext) return 'plaintext';
const extensionMap: { [key: string]: string } = {
// Web
'html': 'html',
'htm': 'html',
'css': 'css',
'scss': 'scss',
'less': 'less',
'js': 'javascript',
'jsx': 'javascript',
'ts': 'typescript',
'tsx': 'typescript',
'json': 'json',
'jsonc': 'json',
// Programming Languages
'py': 'python',
'java': 'java',
'cpp': 'cpp',
'cc': 'cpp',
'h': 'cpp',
'hpp': 'cpp',
'cs': 'csharp',
'go': 'go',
'rs': 'rust',
'rb': 'ruby',
'php': 'php',
'sh': 'shell',
'bash': 'shell',
'zsh': 'shell',
// Markup/Config
'md': 'markdown',
'markdown': 'markdown',
'xml': 'xml',
'svg': 'xml',
'yaml': 'yaml',
'yml': 'yaml',
'ini': 'ini',
'toml': 'ini',
// Other
'sql': 'sql',
'graphql': 'graphql',
'gql': 'graphql',
'dockerfile': 'dockerfile',
'docker': 'dockerfile'
};
return extensionMap[ext] || 'plaintext';
}
export const BlockCode = ({ text, buttonsOnHover, language }: { text: string, buttonsOnHover?: ReactNode, language?: string }) => {
return (<>

View file

@ -70,7 +70,7 @@ const RenderToken = ({ token, nested = false }: { token: Token | string, nested?
if (t.type === "code") {
return <BlockCode
text={t.text}
language={t.lang}
// language={t.lang} // instead use vscode to detect language
buttonsOnHover={<CodeButtonsOnHover diffRepr={t.text} />}
/>
}

View file

@ -18,7 +18,7 @@ import { ErrorDisplay } from './ErrorDisplay.js';
import { OnError, ServiceSendLLMMessageParams } from '../../../../../../../platform/void/common/llmMessageTypes.js';
import { getCmdKey } from '../../../helpers/getCmdKey.js'
import { HistoryInputBox, InputBox } from '../../../../../../../base/browser/ui/inputbox/inputBox.js';
import { VoidCodeEditor, VoidInputBox } from '../util/inputs.js';
import { VoidInputBox } from '../util/inputs.js';
import { ModelDropdown } from '../void-settings-tsx/ModelDropdown.js';
import { chat_systemMessage, chat_prompt } from '../../../prompt/prompts.js';
import { ISidebarStateService } from '../../../sidebarStateService.js';

View file

@ -294,6 +294,7 @@ export const VoidCodeEditor = ({ initValue, language }: { initValue: string, lan
const accessor = useAccessor()
const instantiationService = accessor.get('IInstantiationService')
const modelService = accessor.get('IModelService')
const languageDetectionService = accessor.get('ILanguageDetectionService')
initValue = normalizeIndentation(initValue)
@ -312,7 +313,7 @@ export const VoidCodeEditor = ({ initValue, language }: { initValue: string, lan
alwaysConsumeMouseWheel: false,
vertical: 'auto',
horizontal: 'auto',
verticalScrollbarSize: 0,
// verticalScrollbarSize: 0,
horizontalScrollbarSize: 0,
},
scrollBeyondLastLine: false,
@ -350,7 +351,19 @@ export const VoidCodeEditor = ({ initValue, language }: { initValue: string, lan
onCreateInstance={useCallback((editor: CodeEditorWidget) => {
const model = modelService.createModel(initValue, null)
editor.setModel(model);
model.setLanguage(language ?? 'plaintext')
if (language) {
model.setLanguage(language)
} else {
languageDetectionService.detectLanguage(model.uri).then(detectedLanguage => {
// TODOS:
// once the model has been detected, stop detecting it; currently we detect on every new token which is very slow
// dispose the model; i dont think editor.dispose() does this
model.setLanguage(detectedLanguage ?? 'plaintext')
})
}
const container = editor.getDomNode()
const parentNode = container?.parentElement
@ -374,7 +387,7 @@ export const VoidCodeEditor = ({ initValue, language }: { initValue: string, lan
dispose={useCallback((editor: CodeEditorWidget) => {
editor.dispose();
}, [])}
}, [modelService, languageDetectionService])}
propsFn={useCallback(() => { return [] }, [])}
/>

View file

@ -39,6 +39,7 @@ import { INotificationService } from '../../../../../../../platform/notification
import { IAccessibilityService } from '../../../../../../../platform/accessibility/common/accessibility.js'
import { ILanguageConfigurationService } from '../../../../../../../editor/common/languages/languageConfigurationRegistry.js'
import { ILanguageFeaturesService } from '../../../../../../../editor/common/services/languageFeatures.js'
import { ILanguageDetectionService } from '../../../../../../services/languageDetection/common/languageDetectionWorkerService.js'
@ -145,6 +146,7 @@ export const _registerServices = (accessor: ServicesAccessor) => {
}
const getReactAccessor = (accessor: ServicesAccessor) => {
const reactAccessor = {
IModelService: accessor.get(IModelService),
@ -169,6 +171,7 @@ const getReactAccessor = (accessor: ServicesAccessor) => {
INotificationService: accessor.get(INotificationService),
IAccessibilityService: accessor.get(IAccessibilityService),
ILanguageConfigurationService: accessor.get(ILanguageConfigurationService),
ILanguageDetectionService: accessor.get(ILanguageDetectionService),
ILanguageFeaturesService: accessor.get(ILanguageFeaturesService),
} as const

View file

@ -9,7 +9,7 @@ import { ChatMarkdownRender } from '../markdown/ChatMarkdownRender.js'
const SubtleButton = ({ onClick, text, icon, disabled }: { onClick: () => void, text: string, icon: React.ReactNode, disabled: boolean }) => {
return <div className='flex items-center py-1 px-3 rounded-sm overflow-hidden gap-2 hover:bg-black/10 dark:hover:bg-gray-200/10'>
return <div className='flex items-center px-3 rounded-sm overflow-hidden gap-2 hover:bg-black/10 dark:hover:bg-gray-200/10'>
<button className='flex items-center' disabled={disabled} onClick={onClick}>
{icon}
</button>
@ -19,7 +19,6 @@ const SubtleButton = ({ onClick, text, icon, disabled }: { onClick: () => void,
</div>
}
// models
const RefreshModelButton = ({ providerName }: { providerName: RefreshableProviderName }) => {
const refreshModelState = useRefreshModelState()
@ -59,7 +58,9 @@ const RefreshableModels = () => {
const buttons = refreshableProviderNames.map(providerName => {
if (!settingsState.settingsOfProvider[providerName]._enabled) return null
return <RefreshModelButton key={providerName} providerName={providerName} />
return <div key={providerName} className='pb-4' >
<RefreshModelButton providerName={providerName} />
</div>
})
return <>
@ -346,7 +347,6 @@ export const VoidProviderSettings = () => {
// }
export const VoidFeatureFlagSettings = () => {
const accessor = useAccessor()
const voidSettingsService = accessor.get('IVoidSettingsService')
@ -406,8 +406,8 @@ export const Settings = () => {
<div className={`${tab !== 'models' ? 'hidden' : ''}`}>
<h2 className={`text-3xl mb-2`}>Providers</h2>
<ErrorBoundary>
<VoidFeatureFlagSettings />
<VoidProviderSettings />
<VoidFeatureFlagSettings />
</ErrorBoundary>
<h2 className={`text-3xl mb-2 mt-4`}>Models</h2>