From 4e392bee8ee9bde7fff8bfb03566e621b37cfffb Mon Sep 17 00:00:00 2001 From: Andrew Pareles Date: Mon, 6 Jan 2025 22:25:41 -0800 Subject: [PATCH 01/27] start adding icons for copying files and folders in the explorer --- .../files/browser/views/explorerViewer.ts | 38 ++++++++++++++----- .../files/test/browser/explorerView.test.ts | 7 +++- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/vs/workbench/contrib/files/browser/views/explorerViewer.ts b/src/vs/workbench/contrib/files/browser/views/explorerViewer.ts index 975f494c..216cff20 100644 --- a/src/vs/workbench/contrib/files/browser/views/explorerViewer.ts +++ b/src/vs/workbench/contrib/files/browser/views/explorerViewer.ts @@ -289,6 +289,7 @@ export interface IFileTemplateData { readonly templateDisposables: DisposableStore; readonly elementDisposables: DisposableStore; readonly label: IResourceLabel; + readonly voidLabels: IResourceLabel; readonly container: HTMLElement; readonly contribs: IExplorerFileContribution[]; currentContext?: ExplorerItem; @@ -347,15 +348,24 @@ export class FilesRenderer implements ICompressibleTreeRenderer { + // console.log('ON CLICK', templateData.currentContext?.children) + // }) + const voidLabels = this.labels.create(voidButtonsContainer, { supportHighlights: false, supportIcons: false, }); + voidLabels.element.textContent = 'hi333' + const label = templateDisposables.add(this.labels.create(container, { supportHighlights: true })); templateDisposables.add(label.onDidRender(() => { - try { - if (templateData.currentContext) { - this.updateWidth(templateData.currentContext); - } - } catch (e) { - // noop since the element might no longer be in the tree, no update of width necessary - } + try { if (templateData.currentContext) this.updateWidth(templateData.currentContext); } + catch (e) { /* noop since the element might no longer be in the tree, no update of width necessary*/ } })); const contribs = explorerFileContribRegistry.create(this.instantiationService, container, templateDisposables); @@ -365,10 +375,12 @@ export class FilesRenderer implements ICompressibleTreeRenderer, index: number, templateData: IFileTemplateData): void { const stat = node.element; templateData.currentContext = stat; @@ -382,8 +394,7 @@ export class FilesRenderer implements ICompressibleTreeRenderer c.setResource(undefined)); @@ -477,6 +488,13 @@ export class FilesRenderer implements ICompressibleTreeRenderer { label: { container: label, onDidRender: emitter.event - } + }, + voidLabels: { + container: label, + onDidRender: emitter.event + }, + }, 1, false); ds.add(navigationController); From 0ec0c324b100de48fb36084eba241ee3aeb2c84d Mon Sep 17 00:00:00 2001 From: Andrew Pareles Date: Fri, 31 Jan 2025 20:17:51 -0800 Subject: [PATCH 02/27] void settings opening behavior --- .../src/void-settings-tsx/ModelDropdown.tsx | 3 +-- .../contrib/void/browser/voidSettingsPane.ts | 21 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/ModelDropdown.tsx b/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/ModelDropdown.tsx index bef7f449..bbc6d9d1 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/ModelDropdown.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/ModelDropdown.tsx @@ -40,9 +40,8 @@ const ModelSelectBox = ({ options, featureName }: { options: ModelOption[], feat getOptionDisplayName={(option) => option.selection.modelName} getOptionDropdownName={(option) => option.name} getOptionsEqual={(a, b) => optionsEqual([a], [b])} - className={`text-xs text-void-fg-3 px-1`} + className='text-xs text-void-fg-3 px-1' matchInputWidth={false} - // isMenuPositionFixed={featureName === 'Ctrl+K' ? false : true} /> } // const ModelSelectBox = ({ options, featureName }: { options: ModelOption[], featureName: FeatureName }) => { diff --git a/src/vs/workbench/contrib/void/browser/voidSettingsPane.ts b/src/vs/workbench/contrib/void/browser/voidSettingsPane.ts index c69596eb..e0feaf16 100644 --- a/src/vs/workbench/contrib/void/browser/voidSettingsPane.ts +++ b/src/vs/workbench/contrib/void/browser/voidSettingsPane.ts @@ -8,7 +8,7 @@ import { EditorInput } from '../../../common/editor/editorInput.js'; import * as nls from '../../../../nls.js'; import { EditorExtensions } from '../../../common/editor.js'; import { EditorPane } from '../../../browser/parts/editor/editorPane.js'; -import { IEditorGroup } from '../../../services/editor/common/editorGroupsService.js'; +import { IEditorGroup, IEditorGroupsService } from '../../../services/editor/common/editorGroupsService.js'; import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry.js'; import { IThemeService } from '../../../../platform/theme/common/themeService.js'; import { IStorageService } from '../../../../platform/storage/common/storage.js'; @@ -141,18 +141,27 @@ registerAction2(class extends Action2 { async run(accessor: ServicesAccessor): Promise { const editorService = accessor.get(IEditorService); + const editorGroupService = accessor.get(IEditorGroupsService); + const instantiationService = accessor.get(IInstantiationService); - // close all instances if found - const openEditors = editorService.findEditors(VoidSettingsInput.RESOURCE); - if (openEditors.length > 0) { - await editorService.closeEditors(openEditors); + // if is open, close it + const openEditors = editorService.findEditors(VoidSettingsInput.RESOURCE); // should only have 0 or 1 elements... + if (openEditors.length !== 0) { + const openEditor = openEditors[0].editor + const isCurrentlyOpen = editorService.activeEditor?.resource?.fsPath === openEditor.resource?.fsPath + if (isCurrentlyOpen) + await editorService.closeEditors(openEditors) + else + await editorGroupService.activeGroup.openEditor(openEditor) return; } + // else open it const input = instantiationService.createInstance(VoidSettingsInput); - await editorService.openEditor(input); + + await editorGroupService.activeGroup.openEditor(input); } }) From 3baaf8d6c4d3943d923b051ee4b70f2b0c00bb0a Mon Sep 17 00:00:00 2001 From: Andrew Pareles Date: Fri, 31 Jan 2025 20:24:09 -0800 Subject: [PATCH 03/27] comment --- src/vs/workbench/contrib/void/browser/voidSettingsPane.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vs/workbench/contrib/void/browser/voidSettingsPane.ts b/src/vs/workbench/contrib/void/browser/voidSettingsPane.ts index e0feaf16..0fd8ce2e 100644 --- a/src/vs/workbench/contrib/void/browser/voidSettingsPane.ts +++ b/src/vs/workbench/contrib/void/browser/voidSettingsPane.ts @@ -35,7 +35,7 @@ class VoidSettingsInput extends EditorInput { static readonly ID: string = 'workbench.input.void.settings'; static readonly RESOURCE = URI.from({ // I think this scheme is invalid, it just shuts up TS - scheme: 'void', // Custom scheme for our editor + scheme: 'void', // Custom scheme for our editor (try Schemas.https) path: 'settings' }) readonly resource = VoidSettingsInput.RESOURCE; From 528b8a6b9b80ba1da210e8bbfc1a54e3a0601827 Mon Sep 17 00:00:00 2001 From: Mathew Pareles Date: Tue, 28 Jan 2025 16:22:32 -0800 Subject: [PATCH 04/27] better model onboard --- .gitignore | 1 + src/vs/platform/void/common/llmMessageService.ts | 10 ++++++++++ src/vs/platform/void/common/voidSettingsTypes.ts | 7 +++++++ .../react/src/void-settings-tsx/ModelDropdown.tsx | 14 ++++++++------ 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index fcdb9b6f..17d12e30 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ product.overrides.json *.snap.actual .vscode-test .tmp/ +.tmp2/ .tool-versions diff --git a/src/vs/platform/void/common/llmMessageService.ts b/src/vs/platform/void/common/llmMessageService.ts index fcec27f9..2096209a 100644 --- a/src/vs/platform/void/common/llmMessageService.ts +++ b/src/vs/platform/void/common/llmMessageService.ts @@ -12,6 +12,7 @@ import { createDecorator } from '../../instantiation/common/instantiation.js'; import { Event } from '../../../base/common/event.js'; import { Disposable } from '../../../base/common/lifecycle.js'; import { IVoidSettingsService } from './voidSettingsService.js'; +import { getProvidersWithoutModels } from './voidSettingsTypes.js'; // import { INotificationService } from '../../notification/common/notification.js'; // calls channel to implement features @@ -92,6 +93,15 @@ export class LLMMessageService extends Disposable implements ILLMMessageService // end early if no provider const modelSelection = this.voidSettingsService.state.modelSelectionOfFeature[featureName] + + // throw an error for providers without models + const providersWithoutModels = getProvidersWithoutModels(this.voidSettingsService.state.settingsOfProvider) + if (providersWithoutModels.length !== 0) { + onError({ message: `You haven't added any models for ${providersWithoutModels.join(', ')}.`, fullError: null }) + return null + } + + // throw an error if no models if (modelSelection === null) { onError({ message: 'Please add a Provider in Settings!', fullError: null }) return null diff --git a/src/vs/platform/void/common/voidSettingsTypes.ts b/src/vs/platform/void/common/voidSettingsTypes.ts index 43a29f5b..8caec0be 100644 --- a/src/vs/platform/void/common/voidSettingsTypes.ts +++ b/src/vs/platform/void/common/voidSettingsTypes.ts @@ -185,6 +185,13 @@ export const customSettingNamesOfProvider = (providerName: ProviderName) => { return Object.keys(defaultProviderSettings[providerName]) as CustomSettingName[] } +export const getProvidersWithoutModels = (settingsOfProvider: SettingsOfProvider) => { + return Object.entries(settingsOfProvider) + .filter(([name, provider]) => provider._enabled && provider.models.length === 0) + .map(([name]) => name) +} + + type CommonProviderSettings = { _enabled: boolean | undefined, // undefined initially, computed when user types in all fields diff --git a/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/ModelDropdown.tsx b/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/ModelDropdown.tsx index bbc6d9d1..6b05201f 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/ModelDropdown.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/void-settings-tsx/ModelDropdown.tsx @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------*/ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { FeatureName, featureNames, ModelSelection, modelSelectionsEqual, ProviderName, providerNames } from '../../../../../../../platform/void/common/voidSettingsTypes.js' +import { FeatureName, featureNames, getProvidersWithoutModels, ModelSelection, modelSelectionsEqual, ProviderName, providerNames, SettingsOfProvider } from '../../../../../../../platform/void/common/voidSettingsTypes.js' import { useSettingsState, useRefreshModelState, useAccessor } from '../util/services.js' import { _VoidSelectBox, VoidCustomSelectBox } from '../util/inputs.js' import { SelectBox } from '../../../../../../../base/browser/ui/selectBox/selectBox.js' @@ -12,8 +12,6 @@ import { IconWarning } from '../sidebar-tsx/SidebarChat.js' import { VOID_OPEN_SETTINGS_ACTION_ID, VOID_TOGGLE_SETTINGS_ACTION_ID } from '../../../voidSettingsPane.js' import { ModelOption } from '../../../../../../../platform/void/common/voidSettingsService.js' - - const optionsEqual = (m1: ModelOption[], m2: ModelOption[]) => { if (m1.length !== m2.length) return false for (let i = 0; i < m1.length; i++) { @@ -119,15 +117,19 @@ export const WarningBox = ({ text, onClick, className }: { text: string; onClick export const ModelDropdown = ({ featureName }: { featureName: FeatureName }) => { const settingsState = useSettingsState() + const providersWithMissingModels = getProvidersWithoutModels(settingsState.settingsOfProvider) + const accessor = useAccessor() const commandService = accessor.get('ICommandService') const openSettings = () => { commandService.executeCommand(VOID_OPEN_SETTINGS_ACTION_ID); }; return <> - {settingsState._modelOptions.length === 0 ? - - : + {providersWithMissingModels.length !== 0 ? + + : settingsState._modelOptions.length === 0 ? + + : } } From 41057c20fc7832c7e5e40fa8e08e297bcdefca7f Mon Sep 17 00:00:00 2001 From: Mathew Pareles Date: Wed, 29 Jan 2025 01:19:05 -0800 Subject: [PATCH 05/27] autocomplete type context --- .../void/electron-main/llmMessage/ollama.ts | 2 +- .../void/browser/autocompleteService.ts | 499 +++++++++++++++--- 2 files changed, 418 insertions(+), 83 deletions(-) diff --git a/src/vs/platform/void/electron-main/llmMessage/ollama.ts b/src/vs/platform/void/electron-main/llmMessage/ollama.ts index e76b6186..ac827ffe 100644 --- a/src/vs/platform/void/electron-main/llmMessage/ollama.ts +++ b/src/vs/platform/void/electron-main/llmMessage/ollama.ts @@ -54,10 +54,10 @@ export const sendOllamaFIM: _InternalOllamaFIMMessageFnType = ({ messages, onTex suffix: messages.suffix, options: { stop: messages.stopTokens, + num_predict: 300 // max tokens }, raw: true, stream: true, - // options: { num_predict: parseMaxTokensStr(thisConfig.maxTokens) } // this is max_tokens }) .then(async stream => { _setAborter(() => stream.abort()) diff --git a/src/vs/workbench/contrib/void/browser/autocompleteService.ts b/src/vs/workbench/contrib/void/browser/autocompleteService.ts index eb4d9f7b..39957469 100644 --- a/src/vs/workbench/contrib/void/browser/autocompleteService.ts +++ b/src/vs/workbench/contrib/void/browser/autocompleteService.ts @@ -8,7 +8,7 @@ import { ILanguageFeaturesService } from '../../../../editor/common/services/lan import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; import { ITextModel } from '../../../../editor/common/model.js'; import { Position } from '../../../../editor/common/core/position.js'; -import { InlineCompletion, InlineCompletionContext, LocationLink } from '../../../../editor/common/languages.js'; +import { DocumentSymbol, InlineCompletion, InlineCompletionContext, Location, } from '../../../../editor/common/languages.js'; import { CancellationToken } from '../../../../base/common/cancellation.js'; import { Range } from '../../../../editor/common/core/range.js'; import { ILLMMessageService } from '../../../../platform/void/common/llmMessageService.js'; @@ -155,6 +155,7 @@ type Autocompletion = { llmPromise: Promise | undefined, insertText: string, requestId: string | null, + _newlineCount: number, } const DEBOUNCE_TIME = 500 @@ -163,7 +164,7 @@ const MAX_CACHE_SIZE = 20 const MAX_PENDING_REQUESTS = 2 // postprocesses the result -const joinSpaces = (result: string) => { +const processStartAndEndSpaces = (result: string) => { // trim all whitespace except for a single leading/trailing space // return result.trim() @@ -196,22 +197,26 @@ const removeLeftTabsAndTrimEnds = (s: string): string => { const removeAllWhitespace = (str: string): string => str.replace(/\s+/g, ''); -function isSubsequence({ of, subsequence }: { of: string, subsequence: string }): boolean { - if (subsequence.length === 0) return true; - if (of.length === 0) return false; + + +function getIsSubsequence({ of, subsequence }: { of: string, subsequence: string }): [boolean, string] { + if (subsequence.length === 0) return [true, '']; + if (of.length === 0) return [false, '']; let subsequenceIndex = 0; + let lastMatchChar = ''; for (let i = 0; i < of.length; i++) { if (of[i] === subsequence[subsequenceIndex]) { + lastMatchChar = of[i]; subsequenceIndex++; } if (subsequenceIndex === subsequence.length) { - return true; + return [true, lastMatchChar]; } } - return false; + return [false, lastMatchChar]; } @@ -251,7 +256,6 @@ function getStringUpToUnbalancedClosingParenthesis(s: string, prefix: string): s } - // further trim the autocompletion const postprocessAutocompletion = ({ autocompletionMatchup, autocompletion, prefixAndSuffix }: { autocompletionMatchup: AutocompletionMatchupBounds, autocompletion: Autocompletion, prefixAndSuffix: PrefixAndSuffixInfo }) => { @@ -357,15 +361,24 @@ const toInlineCompletions = ({ autocompletionMatchup, autocompletion, prefixAndS // if we redid the suffix, replace the suffix if (autocompletion.type === 'single-line-redo-suffix') { - if (isSubsequence({ // check that the old text contains the same brackets + symbols as the new text - subsequence: removeAllWhitespace(prefixAndSuffix.suffixToTheRightOfCursor), // old suffix - of: removeAllWhitespace(autocompletion.insertText), // new suffix (note that this should not be `trimmedInsertText`) - })) { + + const oldSuffix = prefixAndSuffix.suffixToTheRightOfCursor + const newSuffix = autocompletion.insertText + + const [isSubsequence, lastMatchingChar] = getIsSubsequence({ // check that the old text contains the same brackets + symbols as the new text + subsequence: removeAllWhitespace(oldSuffix), // old suffix + of: removeAllWhitespace(newSuffix), // new suffix + }) + if (isSubsequence) { rangeToReplace = new Range(position.lineNumber, position.column, position.lineNumber, Number.MAX_SAFE_INTEGER) } else { - // TODO redo the autocompletion - trimmedInsertText = '' // for now set the mismatched text to '' + + const lastMatchupIdx = trimmedInsertText.lastIndexOf(lastMatchingChar) + trimmedInsertText = trimmedInsertText.slice(0, lastMatchupIdx + 1) + const numCharsToReplace = oldSuffix.lastIndexOf(lastMatchingChar) + 1 + rangeToReplace = new Range(position.lineNumber, position.column, position.lineNumber, position.column + numCharsToReplace) + console.log('show____', trimmedInsertText, rangeToReplace) } } @@ -733,12 +746,9 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ // gather relevant context from the code around the user's selection and definitions - const relevantContext = await this._gatherRelevantContextForPosition( - model, - position, - 3, //recursion depth - 1 // number of lines to view in each recursion - ); + const relevantContext = await this._gatherRelevantContextForPosition(model, position); + + console.log('@@---------------------\n' + relevantContext) const { shouldGenerate, predictionType, llmPrefix, llmSuffix, stopTokens } = getCompletionOptions(prefixAndSuffix, relevantContext, justAcceptedAutocompletion) @@ -766,6 +776,7 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ llmPromise: undefined, insertText: '', requestId: null, + _newlineCount: 0, } console.log('BB') @@ -782,28 +793,34 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ stopTokens: stopTokens, }, logging: { loggingName: 'Autocomplete' }, - onText: async ({ fullText }) => { + onText: async ({ fullText, newText }) => { newAutocompletion.insertText = fullText - // if generation doesn't match the prefix for the first few tokens generated, reject it + // count newlines in newText + const numNewlines = newText.match(/\n|\r\n/g)?.length || 0 + newAutocompletion._newlineCount += numNewlines + + // if too many newlines, resolve up to last newline + if (newAutocompletion._newlineCount > 10) { + const lastNewlinePos = fullText.lastIndexOf('\n') + newAutocompletion.insertText = fullText.substring(0, lastNewlinePos) + resolve(newAutocompletion.insertText) + return + } + // if (!getAutocompletionMatchup({ prefix: this._lastPrefix, autocompletion: newAutocompletion })) { - // reject('LLM response did not match user\'s text.') + // reject('LLM response did not match user\'s text.') // } }, onFinalMessage: ({ fullText }) => { console.log('____res: ', JSON.stringify(newAutocompletion.insertText)) - // newAutocompletion.prefix = prefix - // newAutocompletion.suffix = suffix - // newAutocompletion.startTime = Date.now() newAutocompletion.endTime = Date.now() - // newAutocompletion.abortRef = { current: () => { } } newAutocompletion.status = 'finished' - // newAutocompletion.promise = undefined const [text, _] = extractCodeFromRegular({ text: fullText, recentlyAddedTextLen: 0 }) - newAutocompletion.insertText = joinSpaces(text) + newAutocompletion.insertText = processStartAndEndSpaces(text) // handle special case for predicting starting on the next line, add a newline character if (newAutocompletion.type === 'multi-line-start-on-next-line') { @@ -853,84 +870,400 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ } - // helper method to gather ~N lines above and below the user's current line, - // and recursively gather lines around any symbol definitions encountered. + // TODO! Given a user's cursor position, get relevant context. + // algorithm pseudocode: + + // 1. get all relevant symbols (functions, variables, and types) + // 1a. get all symbols that are `numNearbyLines` lines above and below the current position + // eg. if the context is this: + // ``` + // ... + // const addVectors = (a: Vector, b: Vector) => { + // + // ... 100+ LINES OF CODE + // return addVectorsElementWise(a,b, Math.min(a.length, b.length) as NumberType) [[CURSOR]] + // } + // ... + // ``` + // then these are all of the symbols it should consider that are above and below the position: ['addVectorsElementWise', 'Math.min', 'a.length', 'b.length', 'NumberType'] + + // 1b. look at where the parent function is defined and get its nearby symbols `numParentLines` + // ex. + // ``` + // ... + // const addVectors = (a: Vector, b: Vector) => { [[THIS IS THE PARENT FUNCTION]] + // ... 100+ LINES OF CODE + // return addVectorsElementWise(a ,b, Math.min(a.length, b.length)) [[CURSOR IS HERE]] + // } + // ... + // ``` + // the symbols of the parent function are ['const', 'addVectors', 'a', 'Vector', 'b', 'Vector'] + + + // 2. Cmd+Click on each symbol in step 1. (view instances and definitions) + // check that you don't visit the same place twice + // if this location is new, get `` lines above and below this new location and save that string to an array + + // 3. for each of the new positions found in step 2., use step 1 to find all their symbols again. This is the recursive step. + + // use `maxRecursionDepth` to prevent slowness + // set `numNearbyLines` and `numParentLines` to 2 after the first step to increase performance + + // 4. when finished, return snippets.join('\n----------------\n') + + private _docSymbolsCache: { + [docUri: string]: { + version: number; + symbols: DocumentSymbol[]; + }; + } = Object.create(null); + + // For each file, store per-symbol lookups we've done. + // e.g. _symbolLookupCache[docUri][fileVersion]["root"] => Location[] results + private _symbolLookupCache: { + [docUri: string]: { + [version: number]: { + [symbolName: string]: Location[]; + }; + }; + } = Object.create(null); + private async _gatherRelevantContextForPosition( model: ITextModel, position: Position, - recursionDepth: number, - linesAround: number + maxRecursionDepth: number = 3, + numNearbyLines: number = 5, + numParentLines: number = 5, + numSaveLines: number = 10 ): Promise { - // We'll do a BFS-like approach: for each position or definition, gather lines around it, - // then attempt to find the definition of any symbols in that range, up to 'recursionDepth' times. + /**************************************************************************** + * A. Quick Helpers & caches + ****************************************************************************/ + type EditorLocation = import('vs/editor/common/languages').Location; - // A set of "key" strings to avoid repeating the same location or line chunk - const visitedRanges = new Set(); - const collectedSnippets: string[] = []; + const docUri = model.uri.toString(); + const fileVersion = model.getAlternativeVersionId(); + // If you prefer, do a text-based hash or use model.getVersionId() instead. - // A queue of tasks, each being a tuple of: (model, position, depth) - const tasks: Array<{ model: ITextModel, position: Position, depth: number }> = []; - tasks.push({ model, position, depth: recursionDepth }); + // 1) Ensure docSymbols cache + let docSymCache = this._docSymbolsCache[docUri]; + if (!docSymCache || docSymCache.version !== fileVersion) { + docSymCache = { + version: fileVersion, + symbols: await this._getDocumentSymbolsOnce(model) // see helper below + }; + this._docSymbolsCache[docUri] = docSymCache; + } + const allDocumentSymbols = docSymCache.symbols; - const getSnippetAroundLine = (model: ITextModel, lineNumber: number, linesAround: number): string => { - const startLine = Math.max(1, lineNumber - linesAround); - const endLine = Math.min(model.getLineCount(), lineNumber + linesAround); + // 2) Ensure symbol lookup cache + if (!this._symbolLookupCache[docUri]) { + this._symbolLookupCache[docUri] = {}; + } + if (!this._symbolLookupCache[docUri][fileVersion]) { + this._symbolLookupCache[docUri][fileVersion] = {}; + } + const symbolLookupForFile = this._symbolLookupCache[docUri][fileVersion]; + + // Basic numeric clamps + const clampLine = (line: number): number => { + const maxLine = model.getLineCount(); + return Math.max(1, Math.min(line, maxLine)); + }; + + // Return a snippet of lines [start..end] in the document + const snippetForRange = (startLine: number, endLine: number): string => { const lines: string[] = []; - for (let i = startLine; i <= endLine; i++) { - lines.push(model.getLineContent(i)); + for (let ln = startLine; ln <= endLine; ln++) { + lines.push(model.getLineContent(ln)); } return lines.join('\n'); }; - while (tasks.length > 0) { - const { model: currentModel, position: currentPos, depth } = tasks.shift()!; - - if (depth < 0) { - continue; + /**************************************************************************** + * B. Interval-based BFS to gather code blocks without duplication + ****************************************************************************/ + interface Interval { start: number; end: number; } + function addInterval(intervals: Interval[], start: number, end: number) { + // Merge new [start..end] with existing intervals if they overlap or touch + for (let i = 0; i < intervals.length; i++) { + const iv = intervals[i]; + if (!(end < iv.start - 1 || start > iv.end + 1)) { + // Overlaps (or touches); merge + const mergedStart = Math.min(iv.start, start); + const mergedEnd = Math.max(iv.end, end); + intervals.splice(i, 1); // remove old + addInterval(intervals, mergedStart, mergedEnd); // re-run + return; + } } + intervals.push({ start, end }); + } - // Gather snippet around the current line - const snippet = getSnippetAroundLine(currentModel, currentPos.lineNumber, linesAround); - const snippetKey = `${currentModel.uri.toString()}:${currentPos.lineNumber}`; - if (!visitedRanges.has(snippetKey)) { - visitedRanges.add(snippetKey); - collectedSnippets.push(`-- Snippet around line ${currentPos.lineNumber} --\n${snippet}\n`); - } + function intervalsToString(intervals: Interval[]): string { + intervals.sort((a, b) => a.start - b.start); + return intervals + .map(iv => snippetForRange(iv.start, iv.end)) + .join('\n------------------------------\n'); + } - // Attempt to gather definitions for the symbol at this position - // We just pick all definition providers and see if any has a definition - const providers = this._langFeatureService.definitionProvider.ordered(currentModel); - for (const provider of providers) { - try { - const definitions = await provider.provideDefinition(currentModel, currentPos, CancellationToken.None); - if (!definitions) continue; + const intervals: Interval[] = []; + const visitedRanges = new Set(); - // definitions can be a single LocationLink or an array - const defArray: LocationLink[] = Array.isArray(definitions) ? definitions : [definitions]; - for (const def of defArray) { - if (!def.uri) continue; - if (typeof def.range === 'undefined') continue; - const definitionModel = this._modelService.getModel(def.uri); - if (!definitionModel) continue; + function markVisited(s: number, e: number) { visitedRanges.add(`${s}-${e}`); } + function isVisited(s: number, e: number) { return visitedRanges.has(`${s}-${e}`); } - // We'll queue up a new task for that definition range - const defPos = new Position(def.range.startLineNumber, def.range.startColumn); - const defKey = `${def.uri.toString()}:${defPos.lineNumber}`; - if (!visitedRanges.has(defKey)) { - tasks.push({ model: definitionModel, position: defPos, depth: depth - 1 }); + /**************************************************************************** + * C. Compute initial intervals (cursor region, parent symbol region) + ****************************************************************************/ + const lineNumber = position.lineNumber; + const localStart = clampLine(lineNumber - numNearbyLines); + const localEnd = clampLine(lineNumber + numNearbyLines); + + addInterval(intervals, clampLine(localStart - numSaveLines), clampLine(localEnd + numSaveLines)); + markVisited(localStart, localEnd); + + // get parent symbol, add interval for it + const parent = this._findEnclosingSymbol(allDocumentSymbols, lineNumber); + if (parent) { + const pStart = clampLine(parent.range.startLineNumber - numParentLines); + const pEnd = clampLine(parent.range.endLineNumber + numParentLines); + addInterval(intervals, pStart, pEnd); + markVisited(pStart, pEnd); + } + + /**************************************************************************** + * D. BFS data structures + ****************************************************************************/ + interface QItem { start: number; end: number; depth: number; } + const queue: QItem[] = []; + + queue.push({ start: localStart, end: localEnd, depth: 1 }); + if (parent) { + const pStart = clampLine(parent.range.startLineNumber - numParentLines); + const pEnd = clampLine(parent.range.endLineNumber + numParentLines); + queue.push({ start: pStart, end: pEnd, depth: 1 }); + } + + // We'll keep a set of symbols we've done "references + definitions" for: + const visitedSymbolNames = new Set(); + + // Providers + const definitionProviders = this._langFeatureService.definitionProvider.ordered(model); + const referenceProviders = this._langFeatureService.referenceProvider.ordered(model); + + /**************************************************************************** + * E. BFS Loop + ****************************************************************************/ + while (queue.length) { + const { start, end, depth } = queue.shift()!; + if (depth >= maxRecursionDepth) continue; + + // Step 1: Gather all symbols in [start..end] + const regionSyms = this._gatherSymbolsInLineRange(allDocumentSymbols, start, end); + + // For each symbol, do references/defs once per symbol name + for (const sym of regionSyms) { + // If we already resolved that symbolName, skip + const symName = sym.name || ''; + if (!symName) continue; + if (visitedSymbolNames.has(symName)) continue; + visitedSymbolNames.add(symName); + + // If symbol was cached before, skip re-resolving references + if (symbolLookupForFile[symName]) { + // We already have references/definitions => merge them into intervals + const existingLocs = symbolLookupForFile[symName]; + for (const loc of existingLocs) { + const rng = loc.range; + const locStart = clampLine(rng.startLineNumber - numSaveLines); + const locEnd = clampLine(rng.endLineNumber + numSaveLines); + if (!isVisited(locStart, locEnd)) { + markVisited(locStart, locEnd); + addInterval(intervals, locStart, locEnd); + queue.push({ start: locStart, end: locEnd, depth: depth + 1 }); + } + } + continue; + } + + // Not cached => actually ask definitionProviders / referenceProviders + const symPos = this._symbolPosition(sym); // see helper below + let foundLocs: EditorLocation[] = []; + + for (const dp of definitionProviders) { + try { + const defs = await dp.provideDefinition(model, symPos, CancellationToken.None); + if (defs) foundLocs.push(...(Array.isArray(defs) ? defs : [defs])); + } catch {/* ignore */ } + } + for (const rp of referenceProviders) { + try { + const refs = await rp.provideReferences( + model, symPos, { includeDeclaration: true }, CancellationToken.None + ); + if (refs) foundLocs.push(...refs); + } catch {/* ignore */ } + } + + // Filter same-file only + foundLocs = foundLocs.filter(loc => loc.uri.toString() === docUri); + + // Cache them + symbolLookupForFile[symName] = foundLocs; + + // Enqueue each discovered reference/definition + for (const loc of foundLocs) { + const rng = loc.range; + const locStart = clampLine(rng.startLineNumber - numSaveLines); + const locEnd = clampLine(rng.endLineNumber + numSaveLines); + if (!isVisited(locStart, locEnd)) { + markVisited(locStart, locEnd); + addInterval(intervals, locStart, locEnd); + queue.push({ start: locStart, end: locEnd, depth: depth + 1 }); + } + } + } + + // Step 2: Also do naive token-scan for lines in [start..end], + // so e.g. 'root()' calls get recognized if not in docSymbols. + // We can do basically the same "cache symbol name" logic, if you want: + for (let ln = start; ln <= end; ln++) { + const text = model.getLineContent(ln); + const tokens = text.match(/[a-zA-Z_][a-zA-Z0-9_]*/g) || []; + for (const token of tokens) { + if (visitedSymbolNames.has(token)) continue; + visitedSymbolNames.add(token); + + // If cached, merge intervals from cache + if (symbolLookupForFile[token]) { + for (const loc of symbolLookupForFile[token]) { + const rng = loc.range; + const locStart = clampLine(rng.startLineNumber - numSaveLines); + const locEnd = clampLine(rng.endLineNumber + numSaveLines); + if (!isVisited(locStart, locEnd)) { + markVisited(locStart, locEnd); + addInterval(intervals, locStart, locEnd); + queue.push({ start: locStart, end: locEnd, depth: depth + 1 }); + } + } + continue; + } + + // Actually compute definitions/references + const colIdx = text.indexOf(token); + if (colIdx < 0) continue; // should not happen, but just in case + const tokenPos = new Position(ln, colIdx + 1); + let foundLocs: EditorLocation[] = []; + + for (const dp of definitionProviders) { + try { + const defs = await dp.provideDefinition(model, tokenPos, CancellationToken.None); + if (defs) foundLocs.push(...(Array.isArray(defs) ? defs : [defs])); + } catch {/* ignore */ } + } + for (const rp of referenceProviders) { + try { + const refs = await rp.provideReferences( + model, tokenPos, { includeDeclaration: true }, CancellationToken.None + ); + if (refs) foundLocs.push(...refs); + } catch {/* ignore */ } + } + foundLocs = foundLocs.filter(loc => loc.uri.toString() === docUri); + + // Cache them + symbolLookupForFile[token] = foundLocs; + + // Add intervals + for (const loc of foundLocs) { + const rng = loc.range; + const locStart = clampLine(rng.startLineNumber - numSaveLines); + const locEnd = clampLine(rng.endLineNumber + numSaveLines); + if (!isVisited(locStart, locEnd)) { + markVisited(locStart, locEnd); + addInterval(intervals, locStart, locEnd); + queue.push({ start: locStart, end: locEnd, depth: depth + 1 }); } } - } catch (err) { - // If a provider fails, ignore } } } - // Return the joined context - return collectedSnippets.join('\n'); + /**************************************************************************** + * F. Finally, merge intervals and produce final snippet + ****************************************************************************/ + return intervalsToString(intervals); } + /****************************************************************************** + * Additional Helpers + ******************************************************************************/ + private async _getDocumentSymbolsOnce(model: ITextModel): Promise { + const providers = this._langFeatureService.documentSymbolProvider.ordered(model); + let result: DocumentSymbol[] = []; + for (const p of providers) { + try { + const syms = await p.provideDocumentSymbols(model, CancellationToken.None); + if (syms) { + result.push(...syms); + } + } catch {/* ignore */ } + } + return result; + } + + private _findEnclosingSymbol(symbols: DocumentSymbol[], line: number): DocumentSymbol | undefined { + for (const s of symbols) { + if (s.range.startLineNumber <= line && s.range.endLineNumber >= line) { + // Recurse deeper + const child = this._findEnclosingSymbol(s.children || [], line); + return child || s; + } + } + return undefined; + } + + private _symbolPosition(ds: DocumentSymbol): Position { + return new Position(ds.selectionRange.startLineNumber, ds.selectionRange.startColumn); + } + + private _gatherSymbolsInLineRange( + symbols: DocumentSymbol[], + startLine: number, + endLine: number + ): DocumentSymbol[] { + const out: DocumentSymbol[] = []; + for (const ds of symbols) { + if (ds.range.endLineNumber >= startLine && ds.range.startLineNumber <= endLine) { + out.push(ds); + } + if (ds.children?.length) { + out.push(...this._gatherSymbolsInLineRange(ds.children, startLine, endLine)); + } + } + return out; + } + + + + + + + + + + + + + + + + + + + constructor( @ILanguageFeaturesService private _langFeatureService: ILanguageFeaturesService, @ILLMMessageService private readonly _llmMessageService: ILLMMessageService, @@ -966,8 +1299,10 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ // go through cached items and remove matching ones // autocompletion.prefix + autocompletion.insertedText ~== insertedText this._autocompletionsOfDocument[docUriStr].items.forEach((autocompletion: Autocompletion) => { - // const matchup = getAutocompletionMatchup({ prefix, autocompletion }) + + // we can do this more efficiently, I just didn't want to deal with all of the edge cases const matchup = removeAllWhitespace(prefix) === removeAllWhitespace(autocompletion.prefix + autocompletion.insertText) + if (matchup) { console.log('ACCEPT', autocompletion.id) this._lastCompletionAccept = Date.now() From 145fd9614b2b4de61f4da422d54e9fc9a8b952cb Mon Sep 17 00:00:00 2001 From: Mathew Pareles Date: Mon, 3 Feb 2025 18:27:57 -0800 Subject: [PATCH 06/27] changes --- .../void/electron-main/llmMessage/ollama.ts | 3 +- .../void/browser/autocompleteService.ts | 407 +----------------- .../void/browser/contextGatheringService.ts | 354 +++++++++++++++ src/vs/workbench/contrib/void/browser/test.ts | 258 +++++++++++ .../contrib/void/browser/void.contribution.ts | 4 + 5 files changed, 626 insertions(+), 400 deletions(-) create mode 100644 src/vs/workbench/contrib/void/browser/contextGatheringService.ts create mode 100644 src/vs/workbench/contrib/void/browser/test.ts diff --git a/src/vs/platform/void/electron-main/llmMessage/ollama.ts b/src/vs/platform/void/electron-main/llmMessage/ollama.ts index ac827ffe..a41eedb4 100644 --- a/src/vs/platform/void/electron-main/llmMessage/ollama.ts +++ b/src/vs/platform/void/electron-main/llmMessage/ollama.ts @@ -54,7 +54,8 @@ export const sendOllamaFIM: _InternalOllamaFIMMessageFnType = ({ messages, onTex suffix: messages.suffix, options: { stop: messages.stopTokens, - num_predict: 300 // max tokens + num_predict: 300, // max tokens + // repeat_penalty: 1, }, raw: true, stream: true, diff --git a/src/vs/workbench/contrib/void/browser/autocompleteService.ts b/src/vs/workbench/contrib/void/browser/autocompleteService.ts index 39957469..cb7f3099 100644 --- a/src/vs/workbench/contrib/void/browser/autocompleteService.ts +++ b/src/vs/workbench/contrib/void/browser/autocompleteService.ts @@ -8,7 +8,7 @@ import { ILanguageFeaturesService } from '../../../../editor/common/services/lan import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; import { ITextModel } from '../../../../editor/common/model.js'; import { Position } from '../../../../editor/common/core/position.js'; -import { DocumentSymbol, InlineCompletion, InlineCompletionContext, Location, } from '../../../../editor/common/languages.js'; +import { InlineCompletion, InlineCompletionContext, } from '../../../../editor/common/languages.js'; import { CancellationToken } from '../../../../base/common/cancellation.js'; import { Range } from '../../../../editor/common/core/range.js'; import { ILLMMessageService } from '../../../../platform/void/common/llmMessageService.js'; @@ -19,6 +19,7 @@ import { IModelService } from '../../../../editor/common/services/model.js'; import { extractCodeFromRegular } from './helpers/extractCodeFromResult.js'; import { isWindows } from '../../../../base/common/platform.js'; import { registerWorkbenchContribution2, WorkbenchPhase } from '../../../common/contributions.js'; +import { IContextGatheringService } from './contextGatheringService.js'; // The extension this was called from is here - https://github.com/voideditor/void/blob/autocomplete/extensions/void/src/extension/extension.ts @@ -746,11 +747,12 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ // gather relevant context from the code around the user's selection and definitions - const relevantContext = await this._gatherRelevantContextForPosition(model, position); + // const relevantSnippetsList = await this._contextGatheringService.readCachedSnippets(model, position, 3); + const relevantSnippetsList = this._contextGatheringService.getCachedSnippets(); + const relevantSnippets = relevantSnippetsList.map((text) => `${text}`).join('\n-------------------------------\n') + console.log('@@---------------------\n' + relevantSnippets) - console.log('@@---------------------\n' + relevantContext) - - const { shouldGenerate, predictionType, llmPrefix, llmSuffix, stopTokens } = getCompletionOptions(prefixAndSuffix, relevantContext, justAcceptedAutocompletion) + const { shouldGenerate, predictionType, llmPrefix, llmSuffix, stopTokens } = getCompletionOptions(prefixAndSuffix, relevantSnippets, justAcceptedAutocompletion) if (!shouldGenerate) return [] @@ -870,405 +872,12 @@ export class AutocompleteService extends Disposable implements IAutocompleteServ } - // TODO! Given a user's cursor position, get relevant context. - // algorithm pseudocode: - - // 1. get all relevant symbols (functions, variables, and types) - // 1a. get all symbols that are `numNearbyLines` lines above and below the current position - // eg. if the context is this: - // ``` - // ... - // const addVectors = (a: Vector, b: Vector) => { - // - // ... 100+ LINES OF CODE - // return addVectorsElementWise(a,b, Math.min(a.length, b.length) as NumberType) [[CURSOR]] - // } - // ... - // ``` - // then these are all of the symbols it should consider that are above and below the position: ['addVectorsElementWise', 'Math.min', 'a.length', 'b.length', 'NumberType'] - - // 1b. look at where the parent function is defined and get its nearby symbols `numParentLines` - // ex. - // ``` - // ... - // const addVectors = (a: Vector, b: Vector) => { [[THIS IS THE PARENT FUNCTION]] - // ... 100+ LINES OF CODE - // return addVectorsElementWise(a ,b, Math.min(a.length, b.length)) [[CURSOR IS HERE]] - // } - // ... - // ``` - // the symbols of the parent function are ['const', 'addVectors', 'a', 'Vector', 'b', 'Vector'] - - - // 2. Cmd+Click on each symbol in step 1. (view instances and definitions) - // check that you don't visit the same place twice - // if this location is new, get `` lines above and below this new location and save that string to an array - - // 3. for each of the new positions found in step 2., use step 1 to find all their symbols again. This is the recursive step. - - // use `maxRecursionDepth` to prevent slowness - // set `numNearbyLines` and `numParentLines` to 2 after the first step to increase performance - - // 4. when finished, return snippets.join('\n----------------\n') - - private _docSymbolsCache: { - [docUri: string]: { - version: number; - symbols: DocumentSymbol[]; - }; - } = Object.create(null); - - // For each file, store per-symbol lookups we've done. - // e.g. _symbolLookupCache[docUri][fileVersion]["root"] => Location[] results - private _symbolLookupCache: { - [docUri: string]: { - [version: number]: { - [symbolName: string]: Location[]; - }; - }; - } = Object.create(null); - - private async _gatherRelevantContextForPosition( - model: ITextModel, - position: Position, - maxRecursionDepth: number = 3, - numNearbyLines: number = 5, - numParentLines: number = 5, - numSaveLines: number = 10 - ): Promise { - /**************************************************************************** - * A. Quick Helpers & caches - ****************************************************************************/ - type EditorLocation = import('vs/editor/common/languages').Location; - - const docUri = model.uri.toString(); - const fileVersion = model.getAlternativeVersionId(); - // If you prefer, do a text-based hash or use model.getVersionId() instead. - - // 1) Ensure docSymbols cache - let docSymCache = this._docSymbolsCache[docUri]; - if (!docSymCache || docSymCache.version !== fileVersion) { - docSymCache = { - version: fileVersion, - symbols: await this._getDocumentSymbolsOnce(model) // see helper below - }; - this._docSymbolsCache[docUri] = docSymCache; - } - const allDocumentSymbols = docSymCache.symbols; - - // 2) Ensure symbol lookup cache - if (!this._symbolLookupCache[docUri]) { - this._symbolLookupCache[docUri] = {}; - } - if (!this._symbolLookupCache[docUri][fileVersion]) { - this._symbolLookupCache[docUri][fileVersion] = {}; - } - const symbolLookupForFile = this._symbolLookupCache[docUri][fileVersion]; - - // Basic numeric clamps - const clampLine = (line: number): number => { - const maxLine = model.getLineCount(); - return Math.max(1, Math.min(line, maxLine)); - }; - - // Return a snippet of lines [start..end] in the document - const snippetForRange = (startLine: number, endLine: number): string => { - const lines: string[] = []; - for (let ln = startLine; ln <= endLine; ln++) { - lines.push(model.getLineContent(ln)); - } - return lines.join('\n'); - }; - - /**************************************************************************** - * B. Interval-based BFS to gather code blocks without duplication - ****************************************************************************/ - interface Interval { start: number; end: number; } - function addInterval(intervals: Interval[], start: number, end: number) { - // Merge new [start..end] with existing intervals if they overlap or touch - for (let i = 0; i < intervals.length; i++) { - const iv = intervals[i]; - if (!(end < iv.start - 1 || start > iv.end + 1)) { - // Overlaps (or touches); merge - const mergedStart = Math.min(iv.start, start); - const mergedEnd = Math.max(iv.end, end); - intervals.splice(i, 1); // remove old - addInterval(intervals, mergedStart, mergedEnd); // re-run - return; - } - } - intervals.push({ start, end }); - } - - function intervalsToString(intervals: Interval[]): string { - intervals.sort((a, b) => a.start - b.start); - return intervals - .map(iv => snippetForRange(iv.start, iv.end)) - .join('\n------------------------------\n'); - } - - const intervals: Interval[] = []; - const visitedRanges = new Set(); - - function markVisited(s: number, e: number) { visitedRanges.add(`${s}-${e}`); } - function isVisited(s: number, e: number) { return visitedRanges.has(`${s}-${e}`); } - - /**************************************************************************** - * C. Compute initial intervals (cursor region, parent symbol region) - ****************************************************************************/ - const lineNumber = position.lineNumber; - const localStart = clampLine(lineNumber - numNearbyLines); - const localEnd = clampLine(lineNumber + numNearbyLines); - - addInterval(intervals, clampLine(localStart - numSaveLines), clampLine(localEnd + numSaveLines)); - markVisited(localStart, localEnd); - - // get parent symbol, add interval for it - const parent = this._findEnclosingSymbol(allDocumentSymbols, lineNumber); - if (parent) { - const pStart = clampLine(parent.range.startLineNumber - numParentLines); - const pEnd = clampLine(parent.range.endLineNumber + numParentLines); - addInterval(intervals, pStart, pEnd); - markVisited(pStart, pEnd); - } - - /**************************************************************************** - * D. BFS data structures - ****************************************************************************/ - interface QItem { start: number; end: number; depth: number; } - const queue: QItem[] = []; - - queue.push({ start: localStart, end: localEnd, depth: 1 }); - if (parent) { - const pStart = clampLine(parent.range.startLineNumber - numParentLines); - const pEnd = clampLine(parent.range.endLineNumber + numParentLines); - queue.push({ start: pStart, end: pEnd, depth: 1 }); - } - - // We'll keep a set of symbols we've done "references + definitions" for: - const visitedSymbolNames = new Set(); - - // Providers - const definitionProviders = this._langFeatureService.definitionProvider.ordered(model); - const referenceProviders = this._langFeatureService.referenceProvider.ordered(model); - - /**************************************************************************** - * E. BFS Loop - ****************************************************************************/ - while (queue.length) { - const { start, end, depth } = queue.shift()!; - if (depth >= maxRecursionDepth) continue; - - // Step 1: Gather all symbols in [start..end] - const regionSyms = this._gatherSymbolsInLineRange(allDocumentSymbols, start, end); - - // For each symbol, do references/defs once per symbol name - for (const sym of regionSyms) { - // If we already resolved that symbolName, skip - const symName = sym.name || ''; - if (!symName) continue; - if (visitedSymbolNames.has(symName)) continue; - visitedSymbolNames.add(symName); - - // If symbol was cached before, skip re-resolving references - if (symbolLookupForFile[symName]) { - // We already have references/definitions => merge them into intervals - const existingLocs = symbolLookupForFile[symName]; - for (const loc of existingLocs) { - const rng = loc.range; - const locStart = clampLine(rng.startLineNumber - numSaveLines); - const locEnd = clampLine(rng.endLineNumber + numSaveLines); - if (!isVisited(locStart, locEnd)) { - markVisited(locStart, locEnd); - addInterval(intervals, locStart, locEnd); - queue.push({ start: locStart, end: locEnd, depth: depth + 1 }); - } - } - continue; - } - - // Not cached => actually ask definitionProviders / referenceProviders - const symPos = this._symbolPosition(sym); // see helper below - let foundLocs: EditorLocation[] = []; - - for (const dp of definitionProviders) { - try { - const defs = await dp.provideDefinition(model, symPos, CancellationToken.None); - if (defs) foundLocs.push(...(Array.isArray(defs) ? defs : [defs])); - } catch {/* ignore */ } - } - for (const rp of referenceProviders) { - try { - const refs = await rp.provideReferences( - model, symPos, { includeDeclaration: true }, CancellationToken.None - ); - if (refs) foundLocs.push(...refs); - } catch {/* ignore */ } - } - - // Filter same-file only - foundLocs = foundLocs.filter(loc => loc.uri.toString() === docUri); - - // Cache them - symbolLookupForFile[symName] = foundLocs; - - // Enqueue each discovered reference/definition - for (const loc of foundLocs) { - const rng = loc.range; - const locStart = clampLine(rng.startLineNumber - numSaveLines); - const locEnd = clampLine(rng.endLineNumber + numSaveLines); - if (!isVisited(locStart, locEnd)) { - markVisited(locStart, locEnd); - addInterval(intervals, locStart, locEnd); - queue.push({ start: locStart, end: locEnd, depth: depth + 1 }); - } - } - } - - // Step 2: Also do naive token-scan for lines in [start..end], - // so e.g. 'root()' calls get recognized if not in docSymbols. - // We can do basically the same "cache symbol name" logic, if you want: - for (let ln = start; ln <= end; ln++) { - const text = model.getLineContent(ln); - const tokens = text.match(/[a-zA-Z_][a-zA-Z0-9_]*/g) || []; - for (const token of tokens) { - if (visitedSymbolNames.has(token)) continue; - visitedSymbolNames.add(token); - - // If cached, merge intervals from cache - if (symbolLookupForFile[token]) { - for (const loc of symbolLookupForFile[token]) { - const rng = loc.range; - const locStart = clampLine(rng.startLineNumber - numSaveLines); - const locEnd = clampLine(rng.endLineNumber + numSaveLines); - if (!isVisited(locStart, locEnd)) { - markVisited(locStart, locEnd); - addInterval(intervals, locStart, locEnd); - queue.push({ start: locStart, end: locEnd, depth: depth + 1 }); - } - } - continue; - } - - // Actually compute definitions/references - const colIdx = text.indexOf(token); - if (colIdx < 0) continue; // should not happen, but just in case - const tokenPos = new Position(ln, colIdx + 1); - let foundLocs: EditorLocation[] = []; - - for (const dp of definitionProviders) { - try { - const defs = await dp.provideDefinition(model, tokenPos, CancellationToken.None); - if (defs) foundLocs.push(...(Array.isArray(defs) ? defs : [defs])); - } catch {/* ignore */ } - } - for (const rp of referenceProviders) { - try { - const refs = await rp.provideReferences( - model, tokenPos, { includeDeclaration: true }, CancellationToken.None - ); - if (refs) foundLocs.push(...refs); - } catch {/* ignore */ } - } - foundLocs = foundLocs.filter(loc => loc.uri.toString() === docUri); - - // Cache them - symbolLookupForFile[token] = foundLocs; - - // Add intervals - for (const loc of foundLocs) { - const rng = loc.range; - const locStart = clampLine(rng.startLineNumber - numSaveLines); - const locEnd = clampLine(rng.endLineNumber + numSaveLines); - if (!isVisited(locStart, locEnd)) { - markVisited(locStart, locEnd); - addInterval(intervals, locStart, locEnd); - queue.push({ start: locStart, end: locEnd, depth: depth + 1 }); - } - } - } - } - } - - /**************************************************************************** - * F. Finally, merge intervals and produce final snippet - ****************************************************************************/ - return intervalsToString(intervals); - } - - - /****************************************************************************** - * Additional Helpers - ******************************************************************************/ - private async _getDocumentSymbolsOnce(model: ITextModel): Promise { - const providers = this._langFeatureService.documentSymbolProvider.ordered(model); - let result: DocumentSymbol[] = []; - for (const p of providers) { - try { - const syms = await p.provideDocumentSymbols(model, CancellationToken.None); - if (syms) { - result.push(...syms); - } - } catch {/* ignore */ } - } - return result; - } - - private _findEnclosingSymbol(symbols: DocumentSymbol[], line: number): DocumentSymbol | undefined { - for (const s of symbols) { - if (s.range.startLineNumber <= line && s.range.endLineNumber >= line) { - // Recurse deeper - const child = this._findEnclosingSymbol(s.children || [], line); - return child || s; - } - } - return undefined; - } - - private _symbolPosition(ds: DocumentSymbol): Position { - return new Position(ds.selectionRange.startLineNumber, ds.selectionRange.startColumn); - } - - private _gatherSymbolsInLineRange( - symbols: DocumentSymbol[], - startLine: number, - endLine: number - ): DocumentSymbol[] { - const out: DocumentSymbol[] = []; - for (const ds of symbols) { - if (ds.range.endLineNumber >= startLine && ds.range.startLineNumber <= endLine) { - out.push(ds); - } - if (ds.children?.length) { - out.push(...this._gatherSymbolsInLineRange(ds.children, startLine, endLine)); - } - } - return out; - } - - - - - - - - - - - - - - - - - - - constructor( @ILanguageFeaturesService private _langFeatureService: ILanguageFeaturesService, @ILLMMessageService private readonly _llmMessageService: ILLMMessageService, @IEditorService private readonly _editorService: IEditorService, @IModelService private readonly _modelService: IModelService, + @IContextGatheringService private readonly _contextGatheringService: IContextGatheringService, ) { super() diff --git a/src/vs/workbench/contrib/void/browser/contextGatheringService.ts b/src/vs/workbench/contrib/void/browser/contextGatheringService.ts new file mode 100644 index 00000000..c20e8fb8 --- /dev/null +++ b/src/vs/workbench/contrib/void/browser/contextGatheringService.ts @@ -0,0 +1,354 @@ +import { CancellationToken } from '../../../../base/common/cancellation.js'; +import { Position } from '../../../../editor/common/core/position.js'; +import { DocumentSymbol, SymbolKind } from '../../../../editor/common/languages.js'; +import { ITextModel } from '../../../../editor/common/model.js'; +import { ILanguageFeaturesService } from '../../../../editor/common/services/languageFeatures.js'; +import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; +import { Range, IRange } from '../../../../editor/common/core/range.js'; +import { Disposable } from '../../../../base/common/lifecycle.js'; +import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js'; +import { IModelService } from '../../../../editor/common/services/model.js'; +import { ICodeEditorService } from '../../../../editor/browser/services/codeEditorService.js'; +import { URI } from '../../../../base/common/uri.js'; + + +// make sure snippet logic works +// change logic for `visited` to intervals +// atomically set new snippets at end +// throttle cache setting + +interface IVisitedInterval { + uri: string; + startLine: number; + endLine: number; +} + +export interface IContextGatheringService { + readonly _serviceBrand: undefined; + updateCache(model: ITextModel, pos: Position): Promise; + getCachedSnippets(): string[]; +} + +export const IContextGatheringService = createDecorator('contextGatheringService'); + +class ContextGatheringService extends Disposable implements IContextGatheringService { + _serviceBrand: undefined; + private readonly _NUM_LINES = 3; + private readonly _MAX_SNIPPET_LINES = 7; // Reasonable size for context + // Cache holds the most recent list of snippets. + private _cache: string[] = []; + private _snippetIntervals: IVisitedInterval[] = []; + + constructor( + @ILanguageFeaturesService private readonly _langFeaturesService: ILanguageFeaturesService, + @IModelService private readonly _modelService: IModelService, + @ICodeEditorService private readonly _codeEditorService: ICodeEditorService + ) { + super(); + this._modelService.getModels().forEach(model => this._subscribeToModel(model)); + this._register(this._modelService.onModelAdded(model => this._subscribeToModel(model))); + } + + private _subscribeToModel(model: ITextModel): void { + console.log("Subscribing to model:", model.uri.toString()); + this._register(model.onDidChangeContent(() => { + const editor = this._codeEditorService.getFocusedCodeEditor(); + if (editor && editor.getModel() === model) { + const pos = editor.getPosition(); + console.log("updateCache called at position:", pos); + if (pos) { + this.updateCache(model, pos); + } + } + })); + } + + public async updateCache(model: ITextModel, pos: Position): Promise { + const snippets = new Set(); + this._snippetIntervals = []; // Reset intervals for new cache update + + await this._gatherNearbySnippets(model, pos, this._NUM_LINES, 3, snippets, this._snippetIntervals); + await this._gatherParentSnippets(model, pos, this._NUM_LINES, 3, snippets, this._snippetIntervals); + + // Convert to array and filter overlapping snippets + this._cache = Array.from(snippets); + console.log("Cache updated:", this._cache); + } + + public getCachedSnippets(): string[] { + return this._cache; + } + + // Basic snippet extraction. + private _getSnippetForRange(model: ITextModel, range: IRange, numLines: number): string { + const startLine = Math.max(range.startLineNumber - numLines, 1); + const endLine = Math.min(range.endLineNumber + numLines, model.getLineCount()); + + // Enforce maximum snippet size + const totalLines = endLine - startLine + 1; + const adjustedStartLine = totalLines > this._MAX_SNIPPET_LINES + ? endLine - this._MAX_SNIPPET_LINES + 1 + : startLine; + + const snippetRange = new Range(adjustedStartLine, 1, endLine, model.getLineMaxColumn(endLine)); + return this._cleanSnippet(model.getValueInRange(snippetRange)); + } + + private _cleanSnippet(snippet: string): string { + return snippet + .split('\n') + // Remove empty lines and lines with only comments + .filter(line => { + const trimmed = line.trim(); + return trimmed && !/^\/\/+$/.test(trimmed); + }) + // Rejoin with newlines + .join('\n') + // Remove excess whitespace + .trim(); + } + + private _normalizeSnippet(snippet: string): string { + return snippet + // Remove multiple newlines + .replace(/\n{2,}/g, '\n') + // Remove trailing whitespace + .trim(); + } + + private _addSnippetIfNotOverlapping( + model: ITextModel, + range: IRange, + snippets: Set, + visited: IVisitedInterval[] + ): void { + const startLine = range.startLineNumber; + const endLine = range.endLineNumber; + const uri = model.uri.toString(); + + if (!this._isRangeVisited(uri, startLine, endLine, visited)) { + visited.push({ uri, startLine, endLine }); + const snippet = this._normalizeSnippet(this._getSnippetForRange(model, range, this._NUM_LINES)); + if (snippet.length > 0) { + snippets.add(snippet); + } + } + } + + private async _gatherNearbySnippets( + model: ITextModel, + pos: Position, + numLines: number, + depth: number, + snippets: Set, + visited: IVisitedInterval[] + ): Promise { + if (depth <= 0) return; + + const startLine = Math.max(pos.lineNumber - numLines, 1); + const endLine = Math.min(pos.lineNumber + numLines, model.getLineCount()); + const range = new Range(startLine, 1, endLine, model.getLineMaxColumn(endLine)); + + this._addSnippetIfNotOverlapping(model, range, snippets, visited); + + const symbols = await this._getSymbolsNearPosition(model, pos, numLines); + for (const sym of symbols) { + const defs = await this._getDefinitionSymbols(model, sym); + for (const def of defs) { + const defModel = this._modelService.getModel(def.uri); + if (defModel) { + const defPos = new Position(def.range.startLineNumber, def.range.startColumn); + this._addSnippetIfNotOverlapping(defModel, def.range, snippets, visited); + await this._gatherNearbySnippets(defModel, defPos, numLines, depth - 1, snippets, visited); + } + } + } + } + + private async _gatherParentSnippets( + model: ITextModel, + pos: Position, + numLines: number, + depth: number, + snippets: Set, + visited: IVisitedInterval[] + ): Promise { + if (depth <= 0) return; + + const container = await this._findContainerFunction(model, pos); + if (!container) return; + + const containerRange = container.kind === SymbolKind.Method ? container.selectionRange : container.range; + this._addSnippetIfNotOverlapping(model, containerRange, snippets, visited); + + const symbols = await this._getSymbolsNearRange(model, containerRange, numLines); + for (const sym of symbols) { + const defs = await this._getDefinitionSymbols(model, sym); + for (const def of defs) { + const defModel = this._modelService.getModel(def.uri); + if (defModel) { + const defPos = new Position(def.range.startLineNumber, def.range.startColumn); + this._addSnippetIfNotOverlapping(defModel, def.range, snippets, visited); + await this._gatherNearbySnippets(defModel, defPos, numLines, depth - 1, snippets, visited); + } + } + } + + const containerPos = new Position(containerRange.startLineNumber, containerRange.startColumn); + await this._gatherParentSnippets(model, containerPos, numLines, depth - 1, snippets, visited); + } + + private _isRangeVisited(uri: string, startLine: number, endLine: number, visited: IVisitedInterval[]): boolean { + return visited.some(interval => + interval.uri === uri && + !(endLine < interval.startLine || startLine > interval.endLine) + ); + } + + private async _getSymbolsNearPosition(model: ITextModel, pos: Position, numLines: number): Promise { + const startLine = Math.max(pos.lineNumber - numLines, 1); + const endLine = Math.min(pos.lineNumber + numLines, model.getLineCount()); + const range = new Range(startLine, 1, endLine, model.getLineMaxColumn(endLine)); + return this._getSymbolsInRange(model, range); + } + + private async _getSymbolsNearRange(model: ITextModel, range: IRange, numLines: number): Promise { + const centerLine = Math.floor((range.startLineNumber + range.endLineNumber) / 2); + const startLine = Math.max(centerLine - numLines, 1); + const endLine = Math.min(centerLine + numLines, model.getLineCount()); + const searchRange = new Range(startLine, 1, endLine, model.getLineMaxColumn(endLine)); + return this._getSymbolsInRange(model, searchRange); + } + + private async _getSymbolsInRange(model: ITextModel, range: IRange): Promise { + const symbols: DocumentSymbol[] = []; + const providers = this._langFeaturesService.documentSymbolProvider.ordered(model); + for (const provider of providers) { + try { + const result = await provider.provideDocumentSymbols(model, CancellationToken.None); + if (result) { + const flat = this._flattenSymbols(result); + const intersecting = flat.filter(sym => this._rangesIntersect(sym.range, range)); + symbols.push(...intersecting); + } + } catch (e) { + console.warn("Symbol provider error:", e); + } + } + // Also check reference providers. + const refProviders = this._langFeaturesService.referenceProvider.ordered(model); + for (let line = range.startLineNumber; line <= range.endLineNumber; line++) { + const content = model.getLineContent(line); + const words = content.match(/[a-zA-Z_]\w*/g) || []; + for (const word of words) { + const startColumn = content.indexOf(word) + 1; + const pos = new Position(line, startColumn); + if (!this._positionInRange(pos, range)) continue; + for (const provider of refProviders) { + try { + const refs = await provider.provideReferences(model, pos, { includeDeclaration: true }, CancellationToken.None); + if (refs) { + const filtered = refs.filter(ref => this._rangesIntersect(ref.range, range)); + for (const ref of filtered) { + symbols.push({ + name: word, + detail: '', + kind: SymbolKind.Variable, + range: ref.range, + selectionRange: ref.range, + children: [], + tags: [] + }); + } + } + } catch (e) { + console.warn("Reference provider error:", e); + } + } + } + } + return symbols; + } + + private _flattenSymbols(symbols: DocumentSymbol[]): DocumentSymbol[] { + const flat: DocumentSymbol[] = []; + for (const sym of symbols) { + flat.push(sym); + if (sym.children && sym.children.length > 0) { + flat.push(...this._flattenSymbols(sym.children)); + } + } + return flat; + } + + private _rangesIntersect(a: IRange, b: IRange): boolean { + return !( + a.endLineNumber < b.startLineNumber || + a.startLineNumber > b.endLineNumber || + (a.endLineNumber === b.startLineNumber && a.endColumn < b.startColumn) || + (a.startLineNumber === b.endLineNumber && a.endColumn > b.endColumn) + ); + } + + private _positionInRange(pos: Position, range: IRange): boolean { + return pos.lineNumber >= range.startLineNumber && + pos.lineNumber <= range.endLineNumber && + (pos.lineNumber !== range.startLineNumber || pos.column >= range.startColumn) && + (pos.lineNumber !== range.endLineNumber || pos.column <= range.endColumn); + } + + // Get definition symbols for a given symbol. + private async _getDefinitionSymbols(model: ITextModel, symbol: DocumentSymbol): Promise<(DocumentSymbol & { uri: URI })[]> { + const pos = new Position(symbol.range.startLineNumber, symbol.range.startColumn); + const providers = this._langFeaturesService.definitionProvider.ordered(model); + const defs: (DocumentSymbol & { uri: URI })[] = []; + for (const provider of providers) { + try { + const res = await provider.provideDefinition(model, pos, CancellationToken.None); + if (res) { + const links = Array.isArray(res) ? res : [res]; + defs.push(...links.map(link => ({ + name: symbol.name, + detail: symbol.detail, + kind: symbol.kind, + range: link.range, + selectionRange: link.range, + children: [], + tags: symbol.tags || [], + uri: link.uri // Now keeping it as URI instead of converting to string + }))); + } + } catch (e) { + console.warn("Definition provider error:", e); + } + } + return defs; + } + + private async _findContainerFunction(model: ITextModel, pos: Position): Promise { + const searchRange = new Range( + Math.max(pos.lineNumber - 1, 1), 1, + Math.min(pos.lineNumber + 1, model.getLineCount()), + model.getLineMaxColumn(pos.lineNumber) + ); + const symbols = await this._getSymbolsInRange(model, searchRange); + const funcs = symbols.filter(s => + (s.kind === SymbolKind.Function || s.kind === SymbolKind.Method) && + this._positionInRange(pos, s.range) + ); + if (!funcs.length) return null; + return funcs.reduce((innermost, current) => { + if (!innermost) return current; + const moreInner = + (current.range.startLineNumber > innermost.range.startLineNumber || + (current.range.startLineNumber === innermost.range.startLineNumber && + current.range.startColumn > innermost.range.startColumn)) && + (current.range.endLineNumber < innermost.range.endLineNumber || + (current.range.endLineNumber === innermost.range.endLineNumber && + current.range.endColumn < innermost.range.endColumn)); + return moreInner ? current : innermost; + }, null as DocumentSymbol | null); + } +} + +registerSingleton(IContextGatheringService, ContextGatheringService, InstantiationType.Eager); diff --git a/src/vs/workbench/contrib/void/browser/test.ts b/src/vs/workbench/contrib/void/browser/test.ts new file mode 100644 index 00000000..c34d1edf --- /dev/null +++ b/src/vs/workbench/contrib/void/browser/test.ts @@ -0,0 +1,258 @@ +const root = () => { } + +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// + + + + + + +type ChangeType = { name: string; age: number }; + + + +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// + + + + + + +type EditType = { name: string; age: number }; + + + + +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// + +const fn = (params: { edit: EditType, change: ChangeType }) => { + + + + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + + + // + const r = root() + + + + + + + + + + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + + + + + + + +} + + + +const testVar = 5; + + + + +class MyClass { + + + + + + + + + + + + + + + + + + + + + + private test1(t: EditType) { + // + // + // + //// + // + // + //// + // + // + //// + // + // + //// + // + // + //// + // + // + //// + // + // + //// + // + // + //// + // + // + //// + // + // + //// + //x + // + // + + + const x = 1 + + + + + + + } +} + + +const diff --git a/src/vs/workbench/contrib/void/browser/void.contribution.ts b/src/vs/workbench/contrib/void/browser/void.contribution.ts index ebb17358..da87ac17 100644 --- a/src/vs/workbench/contrib/void/browser/void.contribution.ts +++ b/src/vs/workbench/contrib/void/browser/void.contribution.ts @@ -21,6 +21,10 @@ import './chatThreadService.js' // register Autocomplete import './autocompleteService.js' +// register Context services +import './contextGatheringService.js' +import './contextUserChangesService.js' + // settings pane import './voidSettingsPane.js' From 8f79222b8be40129f4189b514dfd784fb338d5d5 Mon Sep 17 00:00:00 2001 From: Mathew Pareles Date: Mon, 3 Feb 2025 18:45:25 -0800 Subject: [PATCH 07/27] fix spacing --- .../contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx b/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx index 0ea5febb..d6e48017 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx @@ -619,6 +619,7 @@ export const SidebarChat = () => { overflow-x-hidden overflow-y-auto py-4 + ${prevMessagesHTML.length === 0 && !messageSoFar ? 'hidden' : ''} `} style={{ maxHeight: sidebarDimensions.height - historyDimensions.height - formDimensions.height - 36 }} // the height of the previousMessages is determined by all other heights > From 5c4753555e53e3d4304feabdb7c3bbf8cc63efda Mon Sep 17 00:00:00 2001 From: Mathew Pareles Date: Mon, 3 Feb 2025 21:06:29 -0800 Subject: [PATCH 08/27] chatbubble add edit button (need to refactor InputBox to finish) --- .../react/src/sidebar-tsx/SidebarChat.tsx | 146 +++++++++--------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx b/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx index d6e48017..1854d6cf 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx @@ -21,6 +21,7 @@ import { useScrollbarStyles } from '../util/useScrollbarStyles.js'; import { VOID_CTRL_L_ACTION_ID } from '../../../actionIDs.js'; import { filenameToVscodeLanguage } from '../../../helpers/detectLanguage.js'; import { VOID_OPEN_SETTINGS_ACTION_ID } from '../../../voidSettingsPane.js'; +import { Pencil } from 'lucide-react'; export const IconX = ({ size, className = '', ...props }: { size: number, className?: string } & React.SVGProps) => { @@ -428,89 +429,88 @@ export const SelectedFiles = ( } - -const ChatBubble_ = ({ isEditMode, isLoading, children, role }: { role: ChatMessage['role'], children: React.ReactNode, isLoading: boolean, isEditMode: boolean }) => { - - return
-
- {children} - {isLoading && } -
- - {/* edit button */} - {/* {role === 'user' && - { setIsEditMode(v => !v); }} - /> - } */} -
-} - - +type ChatBubbleMode = 'display' | 'edit' const ChatBubble = ({ chatMessage, isLoading }: { chatMessage: ChatMessage, isLoading?: boolean, }) => { const role = chatMessage.role // edit mode state - const [isEditMode, setIsEditMode] = useState(false) - + const [mode, setMode] = useState('display') + const [editText, setEditText] = useState(chatMessage.displayContent ?? '') + const [isHovered, setIsHovered] = useState(false) if (!chatMessage.content && !isLoading) { // don't show if empty and not loading (if loading, want to show) return null } + // set chat bubble contents let chatbubbleContents: React.ReactNode - if (role === 'user') { - chatbubbleContents = <> - - {chatMessage.displayContent} - - {/* {!isEditMode ? chatMessage.displayContent : <>} */} - {/* edit mode content */} - {/* TODO this should be the same input box as in the Sidebar */} - {/*