Elgato_dark/void
1
0
Fork 0
mirror of https://github.com/voideditor/void synced 2026-05-24 09:58:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

jitter changes

Browse source
This commit is contained in:
Andrew Pareles 2025-01-12 01:58:48 -08:00
parent a69b99b562
commit 8da71dc1b7
7 changed files with 243 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
CHANGELOG.md Normal file
View file

@ -0,0 +1,23 @@
1/11/24 - release
Added
- delete the Void extension, and move entirely inside the VS Code codebase.
- model selection
- model fetching with .list()
- We switched from the MIT License to to the Apache 2.0 License.
- New diff algorithm for computing and streaming diffs.
- Streaming a change doesn't jitter the syntax highlighter
- Ctrl+K added!

188
src/vs/workbench/contrib/void/browser/helpers/extractCodeFromResult.ts
View file

@ -3,30 +3,119 @@
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
*--------------------------------------------------------------------------------------*/
class SurroundingsRemover {
readonly originalS: string
i: number
j: number
// modelWasTrainedOnFIM should be false here
export const extractCodeFromFIM = ({ text, midTag, modelWasTrainedOnFIM }: { text: string, midTag: string, modelWasTrainedOnFIM: false }) => {
// string is s[i...j]
/* desired matches
`
``
```
<
<P
<PR
<PRE
<PRE>
<PRE> a
<PRE> a </PRE>
<PRE> a </PRE><
<PRE> a </PRE><M
<PRE> a </PRE><MI
<PRE> a </PRE><MID
<PRE> a </PRE><MID>
constructor(s: string) {
this.originalS = s
this.i = 0
this.j = s.length - 1
}
value() {
return this.originalS.substring(this.i, this.j + 1)
}
<PRE> a <PRE/> ->
*/
// returns whether it removed the whole prefix
removePrefix = (prefix: string): boolean => {
let offset = 0
console.log('prefix', prefix, Math.min(this.j, prefix.length - 1))
while (this.i <= this.j && offset <= prefix.length - 1) {
if (this.originalS.charAt(this.i) !== prefix.charAt(offset))
break
offset += 1
this.i += 1
}
return offset === prefix.length
}
// // removes suffix from right to left
removeSuffix = (suffix: string): boolean => {
// e.g. suffix = <PRE/>, the string is <PRE>hi<P
const s = this.value()
// for every possible prefix of `suffix`, check if string ends with it
for (let len = Math.min(s.length, suffix.length); len >= 1; len -= 1) {
if (s.endsWith(suffix.substring(0, len))) { // the end of the string equals a prefix
this.j -= len
return len === suffix.length
}
}
return false
}
// removeSuffix = (suffix: string): boolean => {
// let offset = 0
// while (this.j >= Math.max(this.i, 0)) {
// if (this.originalS.charAt(this.j) !== suffix.charAt(suffix.length - 1 - offset))
// break
// offset += 1
// this.j -= 1
// }
// return offset === suffix.length
// }
removeFromStartUntil = (until: string, alsoRemoveUntilStr: boolean) => {
const index = this.originalS.indexOf(until, this.i)
if (index === -1) {
this.i = this.j + 1
return false
}
console.log('index', index, until.length)
if (alsoRemoveUntilStr)
this.i = index + until.length
else
this.i = index
return true
}
removeCodeBlock = () => {
const pm = this
const foundCodeBlock = pm.removePrefix('```')
console.log('A', this.i, this.j)
if (!foundCodeBlock) return false
pm.removeFromStartUntil('\n', true) // language
console.log('B', this.i, this.j)
const foundCodeBlockEnd = pm.removeSuffix('```')
if (!foundCodeBlockEnd) return false
console.log('C', this.i, this.j)
pm.removeSuffix('\n')
return true
}
}
export const extractCodeFromRegular = (text: string): string => {
// Match either:
// 1. ```language\n<code>```
// 2. ```<code>```
const pm = new SurroundingsRemover(text)
pm.removeCodeBlock()
const s = pm.value()
return s
}
// Ollama has its own FIM, we should not use this if we use that
export const extractCodeFromFIM = ({ text, midTag }: { text: string, midTag: string }): string => {
/* ------------- summary of the regex -------------
[optional ` | `` | ```]
@ -38,35 +127,40 @@ export const extractCodeFromFIM = ({ text, midTag, modelWasTrainedOnFIM }: { tex
[optional ` | `` | ```]
*/
// const regex = /[\s\S]*?(?:`{1,3}\s*([a-zA-Z_]+[\w]*)?[\s\S]*?)?<MID>([\s\S]*?)(?:<\/MID>|`{1,3}|$)/;
const regex = new RegExp(
`[\\s\\S]*?(?:\`{1,3}\\s*([a-zA-Z_]+[\\w]*)?[\\s\\S]*?)?<${midTag}>([\\s\\S]*?)(?:</${midTag}>|\`{1,3}|$)`,
''
);
const match = text.match(regex);
if (match) {
const [_, languageName, codeBetweenMidTags] = match;
return [languageName, codeBetweenMidTags] as const
const pm = new SurroundingsRemover(text)
console.log('ORIGIINAL CODE', text)
pm.removeCodeBlock()
console.log('D', pm.i, pm.j)
const foundMid = pm.removePrefix(`<${midTag}>`)
console.log('E', midTag, pm.i, pm.j)
if (foundMid) {
pm.removeSuffix(`</${midTag}>`)
console.log('F', pm.i, pm.j)
} else {
return [undefined, extractCodeFromRegular(text)] as const
}
const s = pm.value()
return s
// // const regex = /[\s\S]*?(?:`{1,3}\s*([a-zA-Z_]+[\w]*)?[\s\S]*?)?<MID>([\s\S]*?)(?:<\/MID>|`{1,3}|$)/;
// const regex = new RegExp(
// `[\\s\\S]*?(?:\`{1,3}\\s*([a-zA-Z_]+[\\w]*)?[\\s\\S]*?)?<${midTag}>([\\s\\S]*?)(?:</${midTag}>|\`{1,3}|$)`,
// ''
// );
// const match = text.match(regex);
// if (match) {
// const [_, languageName, codeBetweenMidTags] = match;
// return [languageName, codeBetweenMidTags] as const
// } else {
// return [undefined, extractCodeFromRegular(text)] as const
// }
}
export const extractCodeFromRegular = (result: string) => {
// Match either:
// 1. ```language\n<code>```
// 2. ```<code>```
const match = result.match(/```(?:\w+\n)?([\s\S]*?)```|```([\s\S]*?)```/);
if (!match) {
return result;
}
// Return whichever group matched (non-empty)
return match[1] ?? match[2] ?? result;
}

52
src/vs/workbench/contrib/void/browser/inlineDiffsService.ts
View file

@ -13,7 +13,7 @@ import { ICodeEditorService } from '../../../../editor/browser/services/codeEdit
// import { throttle } from '../../../../base/common/decorators.js';
import { ComputedDiff, findDiffs } from './helpers/findDiffs.js';
import { EndOfLinePreference, IModelDecorationOptions, ITextModel } from '../../../../editor/common/model.js';
import { IRange } from '../../../../editor/common/core/range.js';
import { IRange, Range } from '../../../../editor/common/core/range.js';
import { registerColor } from '../../../../platform/theme/common/colorUtils.js';
import { Color, RGBA } from '../../../../base/common/color.js';
import { IModelService } from '../../../../editor/common/services/model.js';
@ -29,7 +29,6 @@ import { URI } from '../../../../base/common/uri.js';
import { IConsistentEditorItemService, IConsistentItemService } from './helperServices/consistentItemService.js';
import { ctrlKStream_prefixAndSuffix, ctrlKStream_prompt, ctrlKStream_systemMessage, ctrlLStream_prompt, ctrlLStream_systemMessage, defaultFimTags } from './prompt/prompts.js';
import { ILLMMessageService } from '../../../../platform/void/common/llmMessageService.js';
import { IPosition } from '../../../../editor/common/core/position.js';
import { mountCtrlK } from '../browser/react/out/quick-edit-tsx/index.js'
import { QuickEditPropsType } from './quickEditActions.js';
@ -38,6 +37,9 @@ import { LLMMessage } from '../../../../platform/void/common/llmMessageTypes.js'
import { IModelContentChangedEvent } from '../../../../editor/common/textModelEvents.js';
import { extractCodeFromFIM, extractCodeFromRegular } from './helpers/extractCodeFromResult.js';
import { IMetricsService } from '../../../../platform/void/common/metricsService.js';
import { IEditorWorkerService } from '../../../../editor/common/services/editorWorker.js';
import { InlineDecorationType } from '../../../../editor/common/viewModel.js';
import { filenameToVscodeLanguage } from './helpers/detectLanguage.js';
const configOfBG = (color: Color) => {
return { dark: color, light: color, hcDark: color, hcLight: color, }
@ -200,6 +202,7 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
@IInstantiationService private readonly _instantiationService: IInstantiationService,
@IConsistentEditorItemService private readonly _consistentEditorItemService: IConsistentEditorItemService,
@IMetricsService private readonly _metricsService: IMetricsService,
@IEditorWorkerService private readonly _editorWorkerService: IEditorWorkerService,
) {
super();
@ -451,7 +454,13 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
const lines = redText.split('\n');
const lineTokens = lines.map(line => LineTokens.createFromTextAndMetadata([{ text: line, metadata: 0 }], this._langService.languageIdCodec));
const source = new LineSource(lineTokens, lines.map(() => null), false, false)
const result = renderLines(source, renderOptions, [], domNode);
const result = renderLines(source, renderOptions, [
{ // add dummy so it doesn't highlight in red
range: Range.lift({ startLineNumber: 1, startColumn: 1, endLineNumber: Number.MAX_SAFE_INTEGER, endColumn: Number.MAX_SAFE_INTEGER }),
inlineClassName: '',
type: InlineDecorationType.Regular
}
], domNode);
const viewZone: IViewZone = {
// afterLineNumber: computedDiff.startLine - 1,
@ -514,8 +523,9 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
}
return model
}
private _readURI(uri: URI): string | null {
return this._getModel(uri)?.getValue(EndOfLinePreference.LF) ?? null
private _readURI(uri: URI, range?: IRange): string | null {
if (!range) return this._getModel(uri)?.getValue(EndOfLinePreference.LF) ?? null
else return this._getModel(uri)?.getValueInRange(range, EndOfLinePreference.LF) ?? null
}
private _getNumLines(uri: URI): number | null {
return this._getModel(uri)?.getLineCount() ?? null
@ -529,13 +539,19 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
}
weAreWriting = false
private _writeText(uri: URI, text: string, range: IRange, { shouldRealignDiffAreas }: { shouldRealignDiffAreas: boolean }) {
private async _writeText(uri: URI, text: string, range: IRange, { shouldRealignDiffAreas }: { shouldRealignDiffAreas: boolean }) {
const model = this._getModel(uri)
if (!model) return
const uriStr = this._readURI(uri, range)
if (!uriStr) return
this.weAreWriting = true
model.applyEdits([{ range, text }]) // applies edits without adding them to undo/redo stack
this.weAreWriting = false
// minimal edits so not so flashy
const edits = await this._editorWorkerService.computeMoreMinimalEdits(uri, [{ range, text }])
if (edits) {
this.weAreWriting = true
model.applyEdits(edits)
this.weAreWriting = false
}
this._onInternalChangeContent(uri, { shouldRealign: shouldRealignDiffAreas && { newText: text, oldRange: range } })
}
@ -813,7 +829,7 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
// @throttle(100)
private _writeDiffZoneLLMText(diffZone: DiffZone, llmText: string, latestCurrentFileEnd: IPosition, newPosition: IPosition) {
private _writeDiffZoneLLMText(diffZone: DiffZone, llmText: string) {
// ----------- 1. Write the new code to the document -----------
// figure out where to highlight based on where the AI is in the stream right now, use the last diff to figure that out
@ -1055,8 +1071,8 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
const { onFinishEdit } = this._addToHistory(uri)
// __TODO__ ctrl+K should use Ollama's FIM method. Also, modelWasTrainedOnFIM should not be a thing
const modelWasTrainedOnFIM = featureName === 'Ctrl+K' ? false : false
// __TODO__ ctrl+K should use Ollama's FIM method.
const ollamaStyleFIM = false
const modelFimTags = defaultFimTags
const adding: Omit<DiffZone, 'diffareaid'> = {
@ -1087,7 +1103,8 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
}
else if (featureName === 'Ctrl+K') {
const { prefix, suffix } = ctrlKStream_prefixAndSuffix({ fullFileStr: currentFileStr, startLine, endLine })
const userContent = ctrlKStream_prompt({ selection: originalCode, userMessage, prefix, suffix, modelWasTrainedOnFIM, fimTags: modelFimTags, uri })
const language = filenameToVscodeLanguage(uri.fsPath) ?? ''
const userContent = ctrlKStream_prompt({ selection: originalCode, userMessage, prefix, suffix, ollamaStyleFIM, fimTags: modelFimTags, language })
console.log('PREFIX:\n', prefix)
console.log('SUFFIX:\n', suffix)
console.log('USER CONTENT:\n', userContent)
@ -1099,9 +1116,6 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
}
else { throw new Error(`featureName ${featureName} is invalid`) }
// __TODO__ make these only move forward
const latestCurrentFileEnd: IPosition = { lineNumber: 1, column: 1 }
const latestOriginalFileStart: IPosition = { lineNumber: 1, column: 1 }
const onDone = () => {
diffZone._streamState = { isStreaming: false, }
@ -1121,8 +1135,8 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
const extractText = (fullText: string) => {
if (featureName === 'Ctrl+K') {
const [_, textSoFar] = extractCodeFromFIM({ text: fullText, midTag: modelFimTags.midTag, modelWasTrainedOnFIM })
return textSoFar
if (ollamaStyleFIM) return fullText
return extractCodeFromFIM({ text: fullText, midTag: modelFimTags.midTag })
}
else if (featureName === 'Ctrl+L') {
return extractCodeFromRegular(fullText)
@ -1135,7 +1149,7 @@ class InlineDiffsService extends Disposable implements IInlineDiffsService {
logging: { loggingName: `startApplying - ${featureName}` },
messages,
onText: ({ newText, fullText }) => {
this._writeDiffZoneLLMText(diffZone, extractText(fullText), latestCurrentFileEnd, latestOriginalFileStart)
this._writeDiffZoneLLMText(diffZone, extractText(fullText))
this._refreshStylesAndDiffsInURI(uri)
},
onFinalMessage: ({ fullText }) => {

7
src/vs/workbench/contrib/void/browser/prompt/prompts.ts
View file

@ -341,12 +341,13 @@ export const defaultFimTags: FimTagsType = {
midTag: 'SELECTION',
}
export const ctrlKStream_prompt = ({ selection, prefix, suffix, userMessage, modelWasTrainedOnFIM, fimTags, uri }: { selection: string, prefix: string, suffix: string, userMessage: string, modelWasTrainedOnFIM: boolean, fimTags: FimTagsType, uri: URI }) => {
export const ctrlKStream_prompt = ({ selection, prefix, suffix, userMessage, fimTags, ollamaStyleFIM, language }:
{ selection: string, prefix: string, suffix: string, userMessage: string, ollamaStyleFIM: boolean, fimTags: FimTagsType, language: string }) => {
const { preTag, sufTag, midTag } = fimTags
const language = filenameToVscodeLanguage(uri.fsPath) ?? ''
if (modelWasTrainedOnFIM) {
if (ollamaStyleFIM) {
// const preTag = 'PRE'
// const sufTag = 'SUF'
// const midTag = 'MID'

2
src/vs/workbench/contrib/void/browser/react/src/markdown/BlockCode.tsx
View file

@ -3,7 +3,7 @@
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
*--------------------------------------------------------------------------------------*/
import { ReactNode } from "react"
import { ReactNode } from 'react'
import { VoidCodeEditor, VoidCodeEditorProps } from '../util/inputs.js';

2
src/vs/workbench/contrib/void/browser/react/src/markdown/ChatMarkdownRender.tsx
View file

@ -88,7 +88,7 @@ const RenderToken = ({ token, nested = false }: { token: Token | string, nested?
if (t.type === "code") {
return <BlockCode
initValue={t.text}
language={t.lang && nameToVscodeLanguage[t.lang]} // use vscode to detect language
language={t.lang === undefined ? undefined : nameToVscodeLanguage[t.lang]} // use vscode to detect language
buttonsOnHover={<CodeButtonsOnHover text={t.text} />}
/>
}

68
src/vs/workbench/contrib/void/browser/react/src/util/inputs.tsx
View file

@ -3,7 +3,7 @@
* Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information.
*--------------------------------------------------------------------------------------*/
import React, { useCallback, useEffect, useRef, useState } from 'react';
import React, { useCallback, useEffect, useId, useRef, useState } from 'react';
import { IInputBoxStyles, InputBox } from '../../../../../../../base/browser/ui/inputbox/inputBox.js';
import { defaultCheckboxStyles, defaultInputBoxStyles, defaultSelectBoxStyles } from '../../../../../../../platform/theme/browser/defaultStyles.js';
import { SelectBox } from '../../../../../../../base/browser/ui/selectBox/selectBox.js';
@ -12,9 +12,7 @@ import { Checkbox } from '../../../../../../../base/browser/ui/toggle/toggle.js'
import { CodeEditorWidget } from '../../../../../../../editor/browser/widget/codeEditor/codeEditorWidget.js'
import { useAccessor } from './services.js';
import { ScrollableElement } from '../../../../../../../base/browser/ui/scrollbar/scrollableElement.js';
import { ModelOption } from '../../../../../../../platform/void/common/voidSettingsService.js';
import { createPortal } from 'react-dom';
import { ITextModel } from '../../../../../../../editor/common/model.js';
// type guard
@ -327,6 +325,7 @@ export const VoidCustomSelectBox = <T extends any>({
{/* Select Button */}
<button
type='button'
ref={buttonRef}
className="flex items-center h-4 bg-transparent whitespace-nowrap hover:brightness-90 w-full"
onClick={() => {
@ -503,8 +502,12 @@ const normalizeIndentation = (code: string): string => {
}
export type VoidCodeEditorProps = { initValue: string, language?: string, maxHeight?: number, showScrollbars?: boolean, placeholderLanguage?: string }
export const VoidCodeEditor = ({ initValue, language, maxHeight, showScrollbars, placeholderLanguage }: VoidCodeEditorProps) => {
const modelOfEditorId: { [id: string]: ITextModel | undefined } = {}
export type VoidCodeEditorProps = { initValue: string, language?: string, maxHeight?: number, showScrollbars?: boolean }
export const VoidCodeEditor = ({ initValue, language, maxHeight, showScrollbars }: VoidCodeEditorProps) => {
initValue = normalizeIndentation(initValue)
// default settings
const MAX_HEIGHT = maxHeight ?? Infinity;
@ -514,10 +517,27 @@ export const VoidCodeEditor = ({ initValue, language, maxHeight, showScrollbars,
const accessor = useAccessor()
const instantiationService = accessor.get('IInstantiationService')
// const languageDetectionService = accessor.get('ILanguageDetectionService')
const modelService = accessor.get('IModelService')
const languageDetectionService = accessor.get('ILanguageDetectionService')
initValue = normalizeIndentation(initValue)
const id = useId()
// these are used to pass to the model creation of modelRef
const initValueRef = useRef(initValue)
const languageRef = useRef(language)
const modelRef = useRef<ITextModel | null>(null)
// if we change the initial value, don't re-render the whole thing, just set it here. same for language
useEffect(() => {
initValueRef.current = initValue
modelRef.current?.setValue(initValue)
}, [initValue])
useEffect(() => {
languageRef.current = language
if (language) modelRef.current?.setLanguage(language)
}, [language])
return <div ref={divRef} className='relative z-0 px-2 py-1 bg-void-bg-3'>
<WidgetComponent
@ -576,31 +596,23 @@ export const VoidCodeEditor = ({ initValue, language, maxHeight, showScrollbars,
{
isSimpleWidget: true,
})
}, [instantiationService])
}
}, [instantiationService])}
onCreateInstance={useCallback((editor: CodeEditorWidget) => {
const model = modelService.createModel(
initValue,
language ? {
languageId: language,
onDidChange: () => ({
dispose: () => { }
})
} : {
languageId: placeholderLanguage ?? '',
onDidChange: () => ({
dispose: () => { }
})
}
);
const model = modelOfEditorId[id] ?? modelService.createModel(
initValueRef.current, {
languageId: languageRef.current ? languageRef.current : '',
onDidChange: (e) => { return { dispose: () => { } } } // no idea why they'd require this
})
modelRef.current = model
editor.setModel(model);
const container = editor.getDomNode()
const parentNode = container?.parentElement
const resize = () => {
const height = editor.getScrollHeight() + 1
if (parentNode) {
const height = Math.min(editor.getScrollHeight() + 1, MAX_HEIGHT);
// const height = Math.min(, MAX_HEIGHT);
parentNode.style.height = `${height}px`;
editor.layout();
}
@ -609,12 +621,12 @@ export const VoidCodeEditor = ({ initValue, language, maxHeight, showScrollbars,
resize()
const disposable = editor.onDidContentSizeChange(() => { resize() });
return [disposable]
}, [modelService, initValue, language])}
return [disposable, model]
}, [modelService])}
dispose={useCallback((editor: CodeEditorWidget) => {
editor.dispose();
}, [modelService, languageDetectionService])}
}, [modelService])}
propsFn={useCallback(() => { return [] }, [])}
/>