From fa43a9f9373fdd013cabe4c9a5b518f3ed54b89b Mon Sep 17 00:00:00 2001 From: Andrew Pareles Date: Fri, 14 Mar 2025 00:20:03 -0700 Subject: [PATCH] language detection --- .../contrib/void/browser/chatThreadService.ts | 6 +- .../contrib/void/browser/editCodeService.ts | 12 +- .../react/src/markdown/ChatMarkdownRender.tsx | 48 ++-- .../react/src/sidebar-tsx/SidebarChat.tsx | 6 +- .../void/browser/react/src/util/inputs.tsx | 4 +- .../void/browser/react/src/util/services.tsx | 4 +- .../contrib/void/browser/sidebarActions.ts | 2 + .../void/common/chatThreadServiceTypes.ts | 2 + .../void/common/helpers/detectLanguage.ts | 174 ------------- .../void/common/helpers/getLanguage.ts | 229 ++++++++++++++++++ .../contrib/void/common/prompt/prompts.ts | 99 ++++---- .../llmMessage/preprocessLLMMessages.ts | 2 + .../llmMessage/sendLLMMessage.impl.ts | 8 +- 13 files changed, 348 insertions(+), 248 deletions(-) delete mode 100644 src/vs/workbench/contrib/void/common/helpers/detectLanguage.ts create mode 100644 src/vs/workbench/contrib/void/common/helpers/getLanguage.ts diff --git a/src/vs/workbench/contrib/void/browser/chatThreadService.ts b/src/vs/workbench/contrib/void/browser/chatThreadService.ts index 04cef415..6cbff796 100644 --- a/src/vs/workbench/contrib/void/browser/chatThreadService.ts +++ b/src/vs/workbench/contrib/void/browser/chatThreadService.ts @@ -204,7 +204,7 @@ class ChatThreadService extends Disposable implements IChatThreadService { @IVoidSettingsService private readonly _settingsService: IVoidSettingsService, @ILanguageFeaturesService private readonly _languageFeaturesService: ILanguageFeaturesService, @ITextModelService private readonly _textModelService: ITextModelService, - @ITerminalToolService private readonly terminalToolService: ITerminalToolService, + @ITerminalToolService private readonly _terminalToolService: ITerminalToolService, ) { super() this.state = { allThreads: {}, currentThreadId: null as unknown as string } // default state @@ -226,7 +226,7 @@ class ChatThreadService extends Disposable implements IChatThreadService { private _convertThreadDataFromStorage(threadsStr: string): ChatThreads { return JSON.parse(threadsStr, (key, value) => { if (value && typeof value === 'object' && value.$mid === 1) { // $mid is the MarshalledId. $mid === 1 means it is a URI - return URI.from(value); + return URI.from(value); // TODO URI.revive instead of this? } return value; }); @@ -400,7 +400,7 @@ class ChatThreadService extends Disposable implements IChatThreadService { // system message const workspaceFolders = this._workspaceContextService.getWorkspace().folders.map(f => f.uri.fsPath) - const terminalIds = this.terminalToolService.listTerminalIds() + const terminalIds = this._terminalToolService.listTerminalIds() const systemMessage = chat_systemMessage(workspaceFolders, terminalIds, chatMode) // all messages so far in the chat history (including tools) diff --git a/src/vs/workbench/contrib/void/browser/editCodeService.ts b/src/vs/workbench/contrib/void/browser/editCodeService.ts index 4fed6693..c4b23325 100644 --- a/src/vs/workbench/contrib/void/browser/editCodeService.ts +++ b/src/vs/workbench/contrib/void/browser/editCodeService.ts @@ -31,7 +31,6 @@ import { mountCtrlK } from './react/out/quick-edit-tsx/index.js' import { QuickEditPropsType } from './quickEditActions.js'; import { IModelContentChangedEvent } from '../../../../editor/common/textModelEvents.js'; import { extractCodeFromFIM, extractCodeFromRegular, ExtractedSearchReplaceBlock, extractSearchReplaceBlocks } from '../common/helpers/extractCodeFromResult.js'; -import { filenameToVscodeLanguage } from '../common/helpers/detectLanguage.js'; import { INotificationService, Severity } from '../../../../platform/notification/common/notification.js'; import { isMacintosh } from '../../../../base/common/platform.js'; import { EditorOption } from '../../../../editor/common/config/editorOptions.js'; @@ -46,6 +45,7 @@ import { IEditCodeService, URIStreamState, AddCtrlKOpts, StartApplyingOpts } fro import { IVoidSettingsService } from '../common/voidSettingsService.js'; import { FeatureName } from '../common/voidSettingsTypes.js'; import { ILanguageService } from '../../../../editor/common/languages/language.js'; +import { getFullLanguage, getLanguageFromModel } from '../common/helpers/getLanguage.js'; // import { IFileService } from '../../../../platform/files/common/files.js'; // import { VSBuffer } from '../../../../base/common/buffer.js'; @@ -756,7 +756,7 @@ class EditCodeService extends Disposable implements IEditCodeService { const fileStr = await this._voidFileService.readFile(uri) if (fileStr === null) return null - const lang = this._languageService.createByFilepathOrFirstLine(uri, fileStr?.split('\n')?.[0]) + const lang = getFullLanguage(this._languageService, { uri, fileContents: fileStr }) const model = this._modelService.createModel(fileStr, lang, uri); return model } @@ -1279,7 +1279,6 @@ class EditCodeService extends Disposable implements IEditCodeService { const c_ = await this._voidFileService.readFile(uri) if (c_ === null) return currentFileStr = c_ - console.log('got curent file', c_.length) const numLines = numLinesOfStr(currentFileStr) @@ -1312,6 +1311,8 @@ class EditCodeService extends Disposable implements IEditCodeService { } const originalCode = currentFileStr.split('\n').slice((startLine - 1), (endLine - 1) + 1).join('\n') + const language = getLanguageFromModel(uri, this._modelService) + let streamRequestIdRef: { current: string | null } = { current: null } // promise that resolves when the apply is done @@ -1346,6 +1347,8 @@ class EditCodeService extends Disposable implements IEditCodeService { this._onDidChangeDiffZoneStreaming.fire({ uri, diffareaid: diffZone.diffareaid }) this._onDidAddOrDeleteDiffZones.fire({ uri }) + + if (from === 'QuickEdit') { const { diffareaid } = opts const ctrlKZone = this.diffAreaOfId[diffareaid] @@ -1359,7 +1362,7 @@ class EditCodeService extends Disposable implements IEditCodeService { let messages: LLMChatMessage[] if (from === 'ClickApply') { - const userContent = rewriteCode_userMessage({ originalCode, applyStr: opts.applyStr, uri }) + const userContent = rewriteCode_userMessage({ originalCode, applyStr: opts.applyStr, language }) messages = [ { role: 'system', content: rewriteCode_systemMessage, }, { role: 'user', content: userContent, } @@ -1373,7 +1376,6 @@ class EditCodeService extends Disposable implements IEditCodeService { const instructions = _mountInfo?.textAreaRef.current?.value ?? '' const { prefix, suffix } = voidPrefixAndSuffix({ fullFileStr: currentFileStr, startLine, endLine }) - const language = filenameToVscodeLanguage(uri.fsPath) ?? '' const userContent = ctrlKStream_userMessage({ selection: originalCode, instructions: instructions, prefix, suffix, isOllamaFIM: false, fimTags: quickEditFIMTags, language }) // type: 'messages', messages = [ diff --git a/src/vs/workbench/contrib/void/browser/react/src/markdown/ChatMarkdownRender.tsx b/src/vs/workbench/contrib/void/browser/react/src/markdown/ChatMarkdownRender.tsx index 4eed66fe..521b552d 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/markdown/ChatMarkdownRender.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/markdown/ChatMarkdownRender.tsx @@ -6,12 +6,13 @@ import React, { JSX, useState } from 'react' import { marked, MarkedToken, Token } from 'marked' import { BlockCode, BlockCodeWithApply } from './BlockCode.js' -import { nameToVscodeLanguage } from '../../../../common/helpers/detectLanguage.js' +import { convertToVscodeLang, getFirstLine, getLanguage } from '../../../../common/helpers/getLanguage.js' import { useApplyButtonHTML } from './ApplyBlockHoverButtons.js' import { useAccessor, useChatThreadsState } from '../util/services.js' import { Range } from '../../../../../../services/search/common/searchExtTypes.js' import { IRange } from '../../../../../../../base/common/range.js' import { ScrollType } from '../../../../../../../editor/common/editorCommon.js' +import { URI } from '../../../../../../../base/common/uri.js' export type ChatMessageLocation = { @@ -101,7 +102,9 @@ const CodespanWithLink = ({ text, rawText, chatMessageLocation }: { text: string export type RenderTokenOptions = { isApplyEnabled?: boolean, isLinkDetectionEnabled?: boolean } -const RenderToken = ({ token, nested, chatMessageLocation, tokenIdx, ...options }: { token: Token | string, nested?: boolean, chatMessageLocation?: ChatMessageLocation, tokenIdx: string, } & RenderTokenOptions): JSX.Element => { +const RenderToken = ({ token, inPTag, chatMessageLocation, tokenIdx, ...options }: { token: Token | string, inPTag?: boolean, chatMessageLocation?: ChatMessageLocation, tokenIdx: string, } & RenderTokenOptions): JSX.Element => { + const accessor = useAccessor() + const languageService = accessor.get('ILanguageService') // deal with built-in tokens first (assume marked token) const t = token as MarkedToken @@ -115,28 +118,41 @@ const RenderToken = ({ token, nested, chatMessageLocation, tokenIdx, ...options } if (t.type === "code") { + const [firstLine, remainingContents] = getFirstLine(t.text) + const firstLineIsURI = URI.isUri(firstLine) - const language = t.lang === undefined ? undefined : nameToVscodeLanguage[t.lang] + let language: string | undefined = undefined + if (t.lang !== undefined) { + // convert markdown language to language that vscode recognizes (eg markdown doesn't know bash but it does know shell) + language = convertToVscodeLang(languageService, t.lang) + } - // TODO user should only be able to apply this when the code has been closed (t.raw ends with "```") + else if (!language) { // if still no lang + if (firstLineIsURI) { // get lang from the uri + const uri = URI.file(firstLine) + language = getLanguage(languageService, { uri, fileContents: remainingContents ?? undefined }) + } + else { // get lang from the contents + language = getLanguage(languageService, { uri: null, fileContents: remainingContents ?? undefined }) + } + } + const contents = firstLineIsURI ? (remainingContents || '') : t.text // exclude first-line URI from contents + // TODO!!! user should only be able to apply this when the code has been closed (t.raw ends with "```") if (options.isApplyEnabled && chatMessageLocation) { - const applyBoxId = getApplyBoxId({ threadId: chatMessageLocation.threadId, messageIdx: chatMessageLocation.messageIdx, tokenIdx: tokenIdx, }) - return } - return } @@ -223,7 +239,7 @@ const RenderToken = ({ token, nested, chatMessageLocation, tokenIdx, ...options return
  • - +
    • } @@ -239,7 +255,7 @@ const RenderToken = ({ token, nested, chatMessageLocation, tokenIdx, ...options )} - + ))} @@ -252,14 +268,15 @@ const RenderToken = ({ token, nested, chatMessageLocation, tokenIdx, ...options {t.tokens.map((token, index) => ( ))} - if (nested) return contents + if (inPTag) return contents return

    {contents} @@ -343,12 +360,13 @@ const RenderToken = ({ token, nested, chatMessageLocation, tokenIdx, ...options ) } -export const ChatMarkdownRender = ({ string, nested = false, chatMessageLocation, ...options }: { string: string, nested?: boolean, chatMessageLocation: ChatMessageLocation | undefined } & RenderTokenOptions) => { + +export const ChatMarkdownRender = ({ string, inPTag = false, chatMessageLocation, ...options }: { string: string, inPTag?: boolean, chatMessageLocation: ChatMessageLocation | undefined } & RenderTokenOptions) => { const tokens = marked.lexer(string); // https://marked.js.org/using_pro#renderer return ( <> {tokens.map((token, index) => ( - + ))} ) diff --git a/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx b/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx index bcb9e5fd..18d3cead 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx @@ -25,11 +25,11 @@ import { VOID_CTRL_L_ACTION_ID } from '../../../actionIDs.js'; import { VOID_OPEN_SETTINGS_ACTION_ID } from '../../../voidSettingsPane.js'; import { FeatureName, isFeatureNameDisabled } from '../../../../../../../workbench/contrib/void/common/voidSettingsTypes.js'; import { WarningBox } from '../void-settings-tsx/WarningBox.js'; -import { filenameToVscodeLanguage } from '../../../../common/helpers/detectLanguage.js'; import { getModelSelectionState, getModelCapabilities } from '../../../../common/modelCapabilities.js'; import { AlertTriangle, ChevronRight, Dot, Pencil, X } from 'lucide-react'; import { ChatMessage, StagingSelectionItem, ToolMessage, ToolRequestApproval } from '../../../../common/chatThreadServiceTypes.js'; import { ToolCallParams, ToolName, ToolNameWithApproval } from '../../../../common/toolsServiceTypes.js'; +import { getLanguageFromModel } from '../../../../common/helpers/getLanguage.js'; @@ -505,6 +505,7 @@ export const SelectedFiles = ( const accessor = useAccessor() const commandService = accessor.get('ICommandService') + const modelService = accessor.get('IModelService') // state for tracking prospective files const { currentUri } = useUriState() @@ -528,6 +529,7 @@ export const SelectedFiles = ( .map(uri => ({ type: 'File', fileURI: uri, + language: getLanguageFromModel(uri, modelService), selectionStr: null, range: null, state: { isOpened: false }, @@ -638,7 +640,7 @@ export const SelectedFiles = ( > diff --git a/src/vs/workbench/contrib/void/browser/react/src/util/inputs.tsx b/src/vs/workbench/contrib/void/browser/react/src/util/inputs.tsx index bf752bc2..069ed703 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/util/inputs.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/util/inputs.tsx @@ -882,9 +882,11 @@ export const VoidCodeEditor = ({ initValue, language, maxHeight, showScrollbars }, [instantiationService])} onCreateInstance={useCallback((editor: CodeEditorWidget) => { + const languageId = languageRef.current ? languageRef.current : 'plaintext' + const model = modelOfEditorId[id] ?? modelService.createModel( initValueRef.current, { - languageId: languageRef.current ? languageRef.current : 'plaintext', + languageId: languageId, onDidChange: (e) => { return { dispose: () => { } } } // no idea why they'd require this }) modelRef.current = model diff --git a/src/vs/workbench/contrib/void/browser/react/src/util/services.tsx b/src/vs/workbench/contrib/void/browser/react/src/util/services.tsx index 4da31c05..5e9f4d4a 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/util/services.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/util/services.tsx @@ -46,6 +46,7 @@ import { IMetricsService } from '../../../../../../../workbench/contrib/void/com import { URI } from '../../../../../../../base/common/uri.js' import { IChatThreadService, ThreadsState, ThreadStreamState } from '../../../chatThreadService.js' import { ITerminalToolService } from '../../../terminalToolService.js' +import { ILanguageService } from '../../../../../../../editor/common/languages/language.js' @@ -225,7 +226,8 @@ const getReactAccessor = (accessor: ServicesAccessor) => { IConfigurationService: accessor.get(IConfigurationService), IPathService: accessor.get(IPathService), IMetricsService: accessor.get(IMetricsService), - ITerminalToolService: accessor.get(ITerminalToolService) + ITerminalToolService: accessor.get(ITerminalToolService), + ILanguageService: accessor.get(ILanguageService), } as const return reactAccessor diff --git a/src/vs/workbench/contrib/void/browser/sidebarActions.ts b/src/vs/workbench/contrib/void/browser/sidebarActions.ts index 799f7777..77993b57 100644 --- a/src/vs/workbench/contrib/void/browser/sidebarActions.ts +++ b/src/vs/workbench/contrib/void/browser/sidebarActions.ts @@ -123,12 +123,14 @@ registerAction2(class extends Action2 { const selection: StagingSelectionItem = !selectionRange || !selectionStr || (selectionRange.startLineNumber > selectionRange.endLineNumber) ? { type: 'File', fileURI: model.uri, + language: model.getLanguageId(), selectionStr: null, range: null, state: { isOpened: false, } } : { type: 'Selection', fileURI: model.uri, + language: model.getLanguageId(), selectionStr: selectionStr, range: selectionRange, state: { isOpened: true, } diff --git a/src/vs/workbench/contrib/void/common/chatThreadServiceTypes.ts b/src/vs/workbench/contrib/void/common/chatThreadServiceTypes.ts index 62051a00..893b4dc1 100644 --- a/src/vs/workbench/contrib/void/common/chatThreadServiceTypes.ts +++ b/src/vs/workbench/contrib/void/common/chatThreadServiceTypes.ts @@ -50,6 +50,7 @@ export type ChatMessage = export type CodeSelection = { type: 'Selection'; fileURI: URI; + language: string; selectionStr: string; range: IRange; state: { @@ -60,6 +61,7 @@ export type CodeSelection = { export type FileSelection = { type: 'File'; fileURI: URI; + language: string; selectionStr: null; range: null; state: { diff --git a/src/vs/workbench/contrib/void/common/helpers/detectLanguage.ts b/src/vs/workbench/contrib/void/common/helpers/detectLanguage.ts deleted file mode 100644 index 9bc3c9bd..00000000 --- a/src/vs/workbench/contrib/void/common/helpers/detectLanguage.ts +++ /dev/null @@ -1,174 +0,0 @@ -/*-------------------------------------------------------------------------------------- - * Copyright 2025 Glass Devtools, Inc. All rights reserved. - * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. - *--------------------------------------------------------------------------------------*/ - -// eg "bash" -> "shell" -export const nameToVscodeLanguage: { [key: string]: string } = { - // Web Technologies - 'html': 'html', - 'css': 'css', - 'scss': 'scss', - 'sass': 'scss', - 'less': 'less', - 'javascript': 'typescript', - 'js': 'typescript', // use more general renderer - 'jsx': 'typescript', - 'typescript': 'typescript', - 'ts': 'typescript', - 'tsx': 'typescript', - 'json': 'json', - 'jsonc': 'json', - - // Programming Languages - 'python': 'python', - 'py': 'python', - 'java': 'java', - 'cpp': 'cpp', - 'c++': 'cpp', - 'c': 'c', - 'csharp': 'csharp', - 'cs': 'csharp', - 'c#': 'csharp', - 'go': 'go', - 'golang': 'go', - 'rust': 'rust', - 'rs': 'rust', - 'ruby': 'ruby', - 'rb': 'ruby', - 'php': 'php', - 'shell': 'shell', - 'bash': 'shell', - 'sh': 'shell', - 'zsh': 'shell', - - // Markup and Config - 'markdown': 'markdown', - 'md': 'markdown', - 'xml': 'xml', - 'svg': 'xml', - 'yaml': 'yaml', - 'yml': 'yaml', - 'ini': 'ini', - 'toml': 'ini', - - // Database and Query Languages - 'sql': 'sql', - 'mysql': 'sql', - 'postgresql': 'sql', - 'graphql': 'graphql', - 'gql': 'graphql', - - // Others - 'dockerfile': 'dockerfile', - 'docker': 'dockerfile', - 'makefile': 'makefile', - 'plaintext': 'plaintext', - 'text': 'plaintext' -}; - - - -// eg ".ts" -> "typescript" -const fileExtensionToVscodeLanguage: { [key: string]: string } = { - // Web - 'html': 'html', - 'htm': 'html', - 'css': 'css', - 'scss': 'scss', - 'less': 'less', - 'js': 'javascript', - 'jsx': 'javascript', - 'ts': 'typescript', - 'tsx': 'typescript', - 'json': 'json', - 'jsonc': 'json', - - // Programming Languages - 'py': 'python', - 'java': 'java', - 'cpp': 'cpp', - 'cc': 'cpp', - 'c': 'c', - 'h': 'cpp', - 'hpp': 'cpp', - 'cs': 'csharp', - 'go': 'go', - 'rs': 'rust', - 'rb': 'ruby', - 'php': 'php', - 'sh': 'shell', - 'bash': 'shell', - 'zsh': 'shell', - - // Markup/Config - 'md': 'markdown', - 'markdown': 'markdown', - 'xml': 'xml', - 'svg': 'xml', - 'yaml': 'yaml', - 'yml': 'yaml', - 'ini': 'ini', - 'toml': 'ini', - - // Other - 'sql': 'sql', - 'graphql': 'graphql', - 'gql': 'graphql', - 'dockerfile': 'dockerfile', - 'docker': 'dockerfile', - 'mk': 'makefile', - - // Config Files and Dot Files - 'npmrc': 'ini', - 'env': 'ini', - 'gitignore': 'ignore', - 'dockerignore': 'ignore', - 'eslintrc': 'json', - 'babelrc': 'json', - 'prettierrc': 'json', - 'stylelintrc': 'json', - 'editorconfig': 'ini', - 'htaccess': 'apacheconf', - 'conf': 'ini', - 'config': 'ini', - - // Package Files - 'package': 'json', - 'package-lock': 'json', - 'gemfile': 'ruby', - 'podfile': 'ruby', - 'rakefile': 'ruby', - - // Build Systems - 'cmake': 'cmake', - 'makefile': 'makefile', - 'gradle': 'groovy', - - // Shell Scripts - 'bashrc': 'shell', - 'zshrc': 'shell', - 'fish': 'shell', - - // Version Control - 'gitconfig': 'ini', - 'hgrc': 'ini', - 'svnconfig': 'ini', - - // Web Server - 'nginx': 'nginx', - - // Misc Config - 'properties': 'properties', - 'cfg': 'ini', - 'reg': 'ini' -}; - - -export function filenameToVscodeLanguage(filename: string): string | undefined { - - const ext = filename.toLowerCase().split('.').pop(); - if (!ext) return undefined; - - return fileExtensionToVscodeLanguage[ext]; -} diff --git a/src/vs/workbench/contrib/void/common/helpers/getLanguage.ts b/src/vs/workbench/contrib/void/common/helpers/getLanguage.ts new file mode 100644 index 00000000..da2c82d1 --- /dev/null +++ b/src/vs/workbench/contrib/void/common/helpers/getLanguage.ts @@ -0,0 +1,229 @@ +// /*-------------------------------------------------------------------------------------- +// * Copyright 2025 Glass Devtools, Inc. All rights reserved. +// * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. +// *--------------------------------------------------------------------------------------*/ + +import { URI } from '../../../../../base/common/uri.js'; +import { ILanguageService } from '../../../../../editor/common/languages/language.js'; +import { IModelService } from '../../../../../editor/common/services/model.js'; + +export const getFirstLine = (content: string): [string, string] | [string, undefined] => { + const newLineIdx = content.indexOf('\r\n') + if (newLineIdx !== -1) { + const A = content.substring(0, newLineIdx + 2) + const B = content.substring(newLineIdx + 2, Infinity); + return [A, B] + } + + const newLineIdx2 = content.indexOf('\n') + if (newLineIdx2 !== -1) { + const A = content.substring(0, newLineIdx2 + 1) + const B = content.substring(newLineIdx2 + 1, Infinity); + return [A, B] + } + + return [content, undefined] +} + + +export function getFullLanguage(languageService: ILanguageService, opts: { uri: URI | null, fileContents: string | undefined }) { + const firstLine = opts.fileContents ? getFirstLine(opts.fileContents)?.[0] : undefined + const fullLang = languageService.createByFilepathOrFirstLine(opts.uri, firstLine) + return fullLang +} + +export function getLanguage(languageService: ILanguageService, opts: { uri: URI | null, fileContents: string | undefined }): string { + return getFullLanguage(languageService, opts).languageId +} + +export function getLanguageFromModel(uri: URI, modelService: IModelService): string { + return modelService.getModel(uri)?.getLanguageId() || '' +} + + + +// conversions +const convertToVoidLanguage = (languageService: ILanguageService, language: string) => { + const { languageId } = languageService.createById(language) + return languageId +} + +export const convertToVscodeLang = (languageService: ILanguageService, markdownLang: string) => { + if (markdownLang in markdownLangToVscodeLang) + return markdownLangToVscodeLang[markdownLang] + return convertToVoidLanguage(languageService, markdownLang) +} + + + + +// // eg "bash" -> "shell" +const markdownLangToVscodeLang: { [key: string]: string } = { + // Web Technologies + 'html': 'html', + 'css': 'css', + 'scss': 'scss', + 'sass': 'scss', + 'less': 'less', + 'javascript': 'typescript', + 'js': 'typescript', // use more general renderer + 'jsx': 'typescript', + 'typescript': 'typescript', + 'ts': 'typescript', + 'tsx': 'typescript', + 'json': 'json', + 'jsonc': 'json', + + // Programming Languages + 'python': 'python', + 'py': 'python', + 'java': 'java', + 'cpp': 'cpp', + 'c++': 'cpp', + 'c': 'c', + 'csharp': 'csharp', + 'cs': 'csharp', + 'c#': 'csharp', + 'go': 'go', + 'golang': 'go', + 'rust': 'rust', + 'rs': 'rust', + 'ruby': 'ruby', + 'rb': 'ruby', + 'php': 'php', + 'shell': 'shellscript', // this is important + 'bash': 'shellscript', + 'sh': 'shellscript', + 'zsh': 'shellscript', + + // Markup and Config + 'markdown': 'markdown', + 'md': 'markdown', + 'xml': 'xml', + 'svg': 'xml', + 'yaml': 'yaml', + 'yml': 'yaml', + 'ini': 'ini', + 'toml': 'ini', + + // Database and Query Languages + 'sql': 'sql', + 'mysql': 'sql', + 'postgresql': 'sql', + 'graphql': 'graphql', + 'gql': 'graphql', + + // Others + 'dockerfile': 'dockerfile', + 'docker': 'dockerfile', + 'makefile': 'makefile', + 'plaintext': 'plaintext', + 'text': 'plaintext' +}; + +// // eg ".ts" -> "typescript" +// const fileExtensionToVscodeLanguage: { [key: string]: string } = { +// // Web +// 'html': 'html', +// 'htm': 'html', +// 'css': 'css', +// 'scss': 'scss', +// 'less': 'less', +// 'js': 'javascript', +// 'jsx': 'javascript', +// 'ts': 'typescript', +// 'tsx': 'typescript', +// 'json': 'json', +// 'jsonc': 'json', + +// // Programming Languages +// 'py': 'python', +// 'java': 'java', +// 'cpp': 'cpp', +// 'cc': 'cpp', +// 'c': 'c', +// 'h': 'cpp', +// 'hpp': 'cpp', +// 'cs': 'csharp', +// 'go': 'go', +// 'rs': 'rust', +// 'rb': 'ruby', +// 'php': 'php', +// 'sh': 'shell', +// 'bash': 'shell', +// 'zsh': 'shell', + +// // Markup/Config +// 'md': 'markdown', +// 'markdown': 'markdown', +// 'xml': 'xml', +// 'svg': 'xml', +// 'yaml': 'yaml', +// 'yml': 'yaml', +// 'ini': 'ini', +// 'toml': 'ini', + +// // Other +// 'sql': 'sql', +// 'graphql': 'graphql', +// 'gql': 'graphql', +// 'dockerfile': 'dockerfile', +// 'docker': 'dockerfile', +// 'mk': 'makefile', + +// // Config Files and Dot Files +// 'npmrc': 'ini', +// 'env': 'ini', +// 'gitignore': 'ignore', +// 'dockerignore': 'ignore', +// 'eslintrc': 'json', +// 'babelrc': 'json', +// 'prettierrc': 'json', +// 'stylelintrc': 'json', +// 'editorconfig': 'ini', +// 'htaccess': 'apacheconf', +// 'conf': 'ini', +// 'config': 'ini', + +// // Package Files +// 'package': 'json', +// 'package-lock': 'json', +// 'gemfile': 'ruby', +// 'podfile': 'ruby', +// 'rakefile': 'ruby', + +// // Build Systems +// 'cmake': 'cmake', +// 'makefile': 'makefile', +// 'gradle': 'groovy', + +// // Shell Scripts +// 'bashrc': 'shell', +// 'zshrc': 'shell', +// 'fish': 'shell', + +// // Version Control +// 'gitconfig': 'ini', +// 'hgrc': 'ini', +// 'svnconfig': 'ini', + +// // Web Server +// 'nginx': 'nginx', + +// // Misc Config +// 'properties': 'properties', +// 'cfg': 'ini', +// 'reg': 'ini' +// }; + + +// export function filenameToVscodeLanguage(filename: string): string | undefined { + + + + +// const ext = filename.toLowerCase().split('.').pop(); +// if (!ext) return undefined; + +// return fileExtensionToVscodeLanguage[ext]; +// } diff --git a/src/vs/workbench/contrib/void/common/prompt/prompts.ts b/src/vs/workbench/contrib/void/common/prompt/prompts.ts index 2d956379..806e8468 100644 --- a/src/vs/workbench/contrib/void/common/prompt/prompts.ts +++ b/src/vs/workbench/contrib/void/common/prompt/prompts.ts @@ -5,8 +5,6 @@ import { URI } from '../../../../../base/common/uri.js'; -import { filenameToVscodeLanguage } from '../helpers/detectLanguage.js'; -import { IModelService } from '../../../../../editor/common/services/model.js'; import { os } from '../helpers/systemInfo.js'; import { IVoidFileService } from '../voidFileService.js'; import { CodeSelection, FileSelection, StagingSelectionItem } from '../chatThreadServiceTypes.js'; @@ -17,8 +15,8 @@ export const tripleTick = ['```', '```'] export const editToolDesc_toolDescription = `\ A high level description of the change you'd like to make in the file. This description will be handed to a dumber, faster model that will quickly apply the change. \ -Typically the best description you can give here is a single code block of the form:\n${tripleTick[0]}\n// ... existing code ...\n{{change 1}}\n// ... existing code ...\n{{change2}}\n// ... existing code ...\n{{change 3}}\n...\n${tripleTick[1]}.\ -Do NOT output the whole file here if possible, and try to write as LITTLE as needed to describe the change.` +Typically the best description you can give here is a single code block of the form:\n${tripleTick[0]}\n// ... existing code ...\n{{change 1}}\n// ... existing code ...\n{{change2}}\n// ... existing code ...\n{{change 3}}\n...\n${tripleTick[1]}. \ +Do NOT output the whole file here if possible, and try to write as LITTLE code as needed to describe the change.` @@ -42,35 +40,37 @@ You will be given tools you can call. - NEVER modify a file outside one of the the user's workspaces without confirmation from the user.`} \ `: `\ -You're allowed to ask for more context. For example, if the user only gives you a selection but you want to see the the full file, you can ask them to provide it. -\ +You're allowed to ask for more context. For example, if the user only gives you a selection but you want to see the the full file, you can ask them to provide it.\ `} ${mode === 'agent' /* code blocks */ ? `\ -If you have a change to make, you should almost always use a tool to edit the file. Even if you don't (e.g. if the user asks you not to), you should still NEVER re-write the entire file for the user. Instead, you should write comments like "// ... existing code" to indicate how to change the existing code. +If you have a change to make, you should almost always use a tool to edit the file. Even if you don't (e.g. if the user asks you not to), you should still NEVER re-write the entire file for the user. Instead, you should write comments like "// ... existing code" to indicate how to change the existing code. \ `: `\ If you think it's appropriate to suggest an edit to a file, then you must describe your suggestion in CODE BLOCK(S) (wrapped in triple backticks). -- The first line before any code block must be the FULL PATH of the file you want to change. If the path does not already exist, it will be created. -- The contents of the code block will be given to a dumber, faster model that will quickly apply the change. +- The first line of the code block must be the FULL PATH of the file you want to change. If the path does not already exist, it will be created. +- The remaining contents of the code block will be given to a dumber, faster model that will quickly apply the change. - Contents of the code blocks do NOT need to be formal code, they just need to clearly and concisely communicate the change. -- Do NOT re-write the entire file in the code block(s). Instead, write comments like "// ... existing code" to indicate how to change the existing code. +- Do NOT re-write the entire file in the code block(s). Instead, write comments like "// ... existing code" to indicate how to change the existing code.`} + +Misc: +- Always wrap any code you produce in triple backticks. \ -`}` +` -type FileSelnLocal = { fileURI: URI, content: string } -const stringifyFileSelection = ({ fileURI, content }: FileSelnLocal) => { +type FileSelnLocal = { fileURI: URI, language: string, content: string } +const stringifyFileSelection = ({ fileURI, language, content }: FileSelnLocal) => { return `\ ${fileURI.fsPath} -${tripleTick[0]}${filenameToVscodeLanguage(fileURI.fsPath) ?? ''} +${tripleTick[0]}${language} ${content} ${tripleTick[1]} ` } -const stringifyCodeSelection = ({ fileURI, selectionStr, range }: CodeSelection) => { +const stringifyCodeSelection = ({ fileURI, language, selectionStr, range }: CodeSelection) => { return `\ ${fileURI.fsPath} (lines ${range.startLineNumber}:${range.endLineNumber}) -${tripleTick[0]}${filenameToVscodeLanguage(fileURI.fsPath) ?? ''} +${tripleTick[0]}${language} ${selectionStr} ${tripleTick[1]} ` @@ -85,14 +85,20 @@ const stringifyFileSelections = async (fileSelections: FileSelection[], voidFile })) return fileSlns.map(sel => stringifyFileSelection(sel)).join('\n') } + + const stringifyCodeSelections = (codeSelections: CodeSelection[]) => { - return codeSelections.map(sel => stringifyCodeSelection(sel)).join('\n') || null + return codeSelections.map(sel => { + stringifyCodeSelection(sel) + }).join('\n') || null } + const stringifySelectionNames = (currSelns: StagingSelectionItem[] | null): string => { if (!currSelns) return '' return currSelns.map(s => `${s.fileURI.fsPath}${s.range ? ` (lines ${s.range.startLineNumber}:${s.range.endLineNumber})` : ''}`).join('\n') } + export const chat_userMessageContent = async (instructions: string, currSelns: StagingSelectionItem[] | null) => { const selnsStr = stringifySelectionNames(currSelns) @@ -103,7 +109,10 @@ export const chat_userMessageContent = async (instructions: string, currSelns: S return str; }; -export const chat_selectionsString = async (prevSelns: StagingSelectionItem[] | null, currSelns: StagingSelectionItem[] | null, voidFileService: IVoidFileService) => { +export const chat_selectionsString = async ( + prevSelns: StagingSelectionItem[] | null, currSelns: StagingSelectionItem[] | null, + voidFileService: IVoidFileService, +) => { // ADD IN FILES AT TOP const allSelections = [...currSelns || [], ...prevSelns || []] @@ -158,9 +167,7 @@ Directions: -export const rewriteCode_userMessage = ({ originalCode, applyStr, uri }: { originalCode: string, applyStr: string, uri: URI }) => { - - const language = filenameToVscodeLanguage(uri.fsPath) ?? '' +export const rewriteCode_userMessage = ({ originalCode, applyStr, language }: { originalCode: string, applyStr: string, language: string }) => { return `\ ORIGINAL_FILE @@ -205,44 +212,44 @@ For example, if the user is asking you to "make this variable a better name", ma - Make sure you give enough context in the code block to apply the changes to the correct location in the code` -export const aiRegex_computeReplacementsForFile_userMessage = async ({ searchClause, replaceClause, fileURI, voidFileService }: { searchClause: string, replaceClause: string, fileURI: URI, modelService: IModelService, voidFileService: IVoidFileService }) => { +// export const aiRegex_computeReplacementsForFile_userMessage = async ({ searchClause, replaceClause, fileURI, voidFileService }: { searchClause: string, replaceClause: string, fileURI: URI, voidFileService: IVoidFileService }) => { - // we may want to do this in batches - const fileSelection: FileSelection = { type: 'File', fileURI, selectionStr: null, range: null, state: { isOpened: false } } +// // we may want to do this in batches +// const fileSelection: FileSelection = { type: 'File', fileURI, selectionStr: null, range: null, state: { isOpened: false } } - const file = await stringifyFileSelections([fileSelection], voidFileService) +// const file = await stringifyFileSelections([fileSelection], voidFileService) - return `\ -## FILE -${file} +// return `\ +// ## FILE +// ${file} -## SEARCH_CLAUSE -Here is what the user is searching for: -${searchClause} +// ## SEARCH_CLAUSE +// Here is what the user is searching for: +// ${searchClause} -## REPLACE_CLAUSE -Here is what the user wants to replace it with: -${replaceClause} +// ## REPLACE_CLAUSE +// Here is what the user wants to replace it with: +// ${replaceClause} -## INSTRUCTIONS -Please return the changes you want to make to the file in a codeblock, or return "no" if you do not want to make changes.` -} +// ## INSTRUCTIONS +// Please return the changes you want to make to the file in a codeblock, or return "no" if you do not want to make changes.` +// } -// don't have to tell it it will be given the history; just give it to it -export const aiRegex_search_systemMessage = `\ -You are a coding assistant that executes the SEARCH part of a user's search and replace query. +// // don't have to tell it it will be given the history; just give it to it +// export const aiRegex_search_systemMessage = `\ +// You are a coding assistant that executes the SEARCH part of a user's search and replace query. -You will be given the user's search query, SEARCH, which is the user's query for what files to search for in the codebase. You may also be given the user's REPLACE query for additional context. +// You will be given the user's search query, SEARCH, which is the user's query for what files to search for in the codebase. You may also be given the user's REPLACE query for additional context. -Output -- Regex query -- Files to Include (optional) -- Files to Exclude? (optional) +// Output +// - Regex query +// - Files to Include (optional) +// - Files to Exclude? (optional) -` +// ` diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts index 2821dfe1..c9388f08 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/preprocessLLMMessages.ts @@ -396,6 +396,8 @@ export const prepareMessages = ({ const { messages: messages3, separateSystemMessageStr } = prepareMessages_systemMessage({ messages: messages2, aiInstructions, supportsSystemMessage }) const { messages: messages4 } = prepareMessages_tools({ messages: messages3, supportsTools }) const { messages: messages5 } = prepareMessages_noEmptyMessage({ messages: messages4 }) + + console.log('MESSAGES!!!', JSON.stringify(messages, null, 2)) return { messages: messages5 as any, separateSystemMessageStr diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.impl.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.impl.ts index abb0bc17..a34535bf 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.impl.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/sendLLMMessage.impl.ts @@ -7,7 +7,6 @@ import Anthropic from '@anthropic-ai/sdk'; import { Ollama } from 'ollama'; import OpenAI, { ClientOptions } from 'openai'; -import { Model as OpenAIModel } from 'openai/resources/models.js'; import { extractReasoningOnFinalMessage, extractReasoningOnTextWrapper } from '../../common/helpers/extractCodeFromResult.js'; import { LLMChatMessage, LLMFIMMessage, ModelListParams, OllamaModelResponse, OnError, OnFinalMessage, OnText } from '../../common/sendLLMMessageTypes.js'; import { defaultProviderSettings, displayInfoOfProviderName, ModelSelectionOptions, ProviderName, SettingsOfProvider } from '../../common/voidSettingsTypes.js'; @@ -236,6 +235,13 @@ const _sendOpenAICompatibleChat = ({ messages: messages_, onText, onFinalMessage } + +type OpenAIModel = { + id: string; + created: number; + object: 'model'; + owned_by: string; +} const _openaiCompatibleList = async ({ onSuccess: onSuccess_, onError: onError_, settingsOfProvider, providerName }: ListParams_Internal) => { const onSuccess = ({ models }: { models: OpenAIModel[] }) => { onSuccess_({ models })