diff --git a/src/vs/workbench/contrib/void/browser/chatThreadService.ts b/src/vs/workbench/contrib/void/browser/chatThreadService.ts index b801a92c..158c7eb8 100644 --- a/src/vs/workbench/contrib/void/browser/chatThreadService.ts +++ b/src/vs/workbench/contrib/void/browser/chatThreadService.ts @@ -1500,9 +1500,9 @@ We only need to do it for files that were edited since `from`, ie files between } }, true) - // when change focused message idx, jump - if (messageIdx !== undefined) - this.jumpToCheckpointBeforeMessageIdx({ threadId, messageIdx, jumpToUserModified: true }) + // // when change focused message idx, jump - do not jump back when click edit, too confusing. + // if (messageIdx !== undefined) + // this.jumpToCheckpointBeforeMessageIdx({ threadId, messageIdx, jumpToUserModified: true }) } // set message.state diff --git a/src/vs/workbench/contrib/void/common/helpers/extractCodeFromResult.ts b/src/vs/workbench/contrib/void/common/helpers/extractCodeFromResult.ts index 3a131461..09e57f04 100644 --- a/src/vs/workbench/contrib/void/common/helpers/extractCodeFromResult.ts +++ b/src/vs/workbench/contrib/void/common/helpers/extractCodeFromResult.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------*/ import { DIVIDER, FINAL, ORIGINAL } from '../prompt/prompts.js' -class SurroundingsRemover { +export class SurroundingsRemover { readonly originalS: string i: number j: number @@ -58,12 +58,13 @@ class SurroundingsRemover { // return offset === suffix.length // } + // either removes all or nothing removeFromStartUntilFullMatch = (until: string, alsoRemoveUntilStr: boolean) => { const index = this.originalS.indexOf(until, this.i) if (index === -1) { - this.i = this.j + 1 - return null + // this.i = this.j + 1 + return false } // console.log('index', index, until.length) diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/extractGrammar.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/extractGrammar.ts index 0fbb1b29..4b589636 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/extractGrammar.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/extractGrammar.ts @@ -3,11 +3,10 @@ * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. *--------------------------------------------------------------------------------------*/ -import { endsWithAnyPrefixOf } from '../../common/helpers/extractCodeFromResult.js' +import { endsWithAnyPrefixOf, SurroundingsRemover } from '../../common/helpers/extractCodeFromResult.js' import { availableTools, InternalToolInfo, ToolName, ToolParamName } from '../../common/prompt/prompts.js' -import { OnFinalMessage, OnText, RawToolCallObj } from '../../common/sendLLMMessageTypes.js' +import { OnFinalMessage, OnText, RawToolCallObj, RawToolParamsObj } from '../../common/sendLLMMessageTypes.js' import { ChatMode } from '../../common/voidSettingsTypes.js' -import { createSaxParser } from './sax.js' // =============== reasoning =============== @@ -137,17 +136,113 @@ export const extractReasoningWrapper = ( // =============== tools =============== -type ToolsState = { - level: 'normal', -} | { - level: 'tool', - toolName: ToolName, - currentToolCall: RawToolCallObj, -} | { - level: 'param', - toolName: ToolName, - paramName: ToolParamName, - currentToolCall: RawToolCallObj, + + +const findPartiallyWrittenToolTagAtEnd = (fullText: string, toolTags: string[]) => { + for (const toolTag of toolTags) { + const foundPrefix = endsWithAnyPrefixOf(fullText, toolTag) + if (foundPrefix) { + return [foundPrefix, toolTag] as const + } + } + return false +} + +const findIndexOfAny = (fullText: string, matches: string[]) => { + for (const str of matches) { + const idx = fullText.indexOf(str); + if (idx !== -1) { + return [idx, str] as const + } + } + return null +} + + +type ToolOfToolName = { [toolName: string]: InternalToolInfo | undefined } +const parseXMLPrefixToToolCall = (toolName: ToolName, str: string, toolOfToolName: ToolOfToolName): RawToolCallObj => { + const paramsObj: RawToolParamsObj = {} + const doneParams: ToolParamName[] = [] + let isDone = false + + const getAnswer = (): RawToolCallObj => { + const ans: RawToolCallObj = { + name: toolName, + rawParams: paramsObj, + doneParams: doneParams, + isDone: isDone + } + return ans + } + + // find first toolName tag + const openToolTag = `<${toolName}>` + let i = str.indexOf(openToolTag) + if (i === -1) return getAnswer() + let j = str.lastIndexOf(``) + if (j === -1) j = Infinity + else isDone = true + + + str = str.substring(i + openToolTag.length, j) + + const pm = new SurroundingsRemover(str) + + const allowedParams = Object.keys(toolOfToolName[toolName]?.params ?? {}) as ToolParamName[] + if (allowedParams.length === 0) return getAnswer() + let latestMatchedOpenParam: null | ToolParamName = null + let n = 0 + while (true) { + n += 1 + if (n > 10) return getAnswer() // just for good measure as this code is early + + // find the param name opening tag + let matchedOpenParam: null | ToolParamName = null + for (const paramName of allowedParams) { + const removed = pm.removeFromStartUntilFullMatch(`<${paramName}>`, true) + if (removed) { + matchedOpenParam = paramName + break + } + } + // if did not find a new param, stop + if (matchedOpenParam === null) { + if (latestMatchedOpenParam !== null) { + paramsObj[latestMatchedOpenParam] += pm.value() + } + return getAnswer() + } + else { + latestMatchedOpenParam = matchedOpenParam + } + + paramsObj[latestMatchedOpenParam] = '' + + // find the param name closing tag + let matchedCloseParam: boolean = false + let paramContents = '' + for (const paramName of allowedParams) { + const i = pm.i + const closeTag = `` + const removed = pm.removeFromStartUntilFullMatch(closeTag, true) + if (removed) { + const i2 = pm.i + paramContents = pm.originalS.substring(i, i2 - closeTag.length) + matchedCloseParam = true + break + } + } + // if did not find a new close tag, stop + if (!matchedCloseParam) { + paramsObj[latestMatchedOpenParam] += pm.value() + return getAnswer() + } + else { + doneParams.push(latestMatchedOpenParam) + } + + paramsObj[latestMatchedOpenParam] += paramContents + } } export const extractToolsWrapper = ( @@ -156,125 +251,18 @@ export const extractToolsWrapper = ( const tools = availableTools(chatMode) if (!tools) return { newOnText: onText, newOnFinalMessage: onFinalMessage } - const toolOfToolName: { [toolName: string]: InternalToolInfo | undefined } = {} + const toolOfToolName: ToolOfToolName = {} + const toolOpenTags = tools.map(t => `<${t.name}>`) for (const t of tools) { toolOfToolName[t.name] = t } // detect , etc let fullText = ''; let trueFullText = '' - const firstToolCallRef: { current: RawToolCallObj | undefined } = { current: undefined } - - let state: ToolsState = { level: 'normal' } - - - const getRawNewText = () => { - return trueFullText.substring(parser.startTagPosition, parser.position + 1) - } - const parser = createSaxParser() - - // when see open tag - parser.onopentag = (node) => { - const rawNewText = getRawNewText() - const tagName = node.name; - console.log('OPENING', tagName) - console.log('state0:', state.level, { toolName: (state as any).toolName, paramName: (state as any).paramName }) - - if (state.level === 'normal') { - if (tagName in toolOfToolName) { // valid toolName - state = { - level: 'tool', - toolName: tagName as ToolName, - currentToolCall: { name: tagName as ToolName, rawParams: {}, doneParams: [], isDone: false } - } - firstToolCallRef.current = state.currentToolCall - } - else { - fullText += rawNewText // count as plaintext - console.log('adding raw a', rawNewText) - - } - } - else if (state.level === 'tool') { - if (tagName in (toolOfToolName[state.toolName]?.params ?? {})) { // valid param - state = { - level: 'param', - toolName: state.toolName, - paramName: tagName as ToolParamName, - currentToolCall: state.currentToolCall, - } - } - else { - // would normally be rawNewText, but we ignore all text inside tools - } - } - else if (state.level === 'param') { // cannot double nest - fullText += rawNewText // count as plaintext - console.log('adding raw b', rawNewText) - - } - - console.log('state1:', state.level, { toolName: (state as any).toolName, paramName: (state as any).paramName }) - - }; - - parser.onclosetag = (tagName) => { - const rawNewText = getRawNewText() - console.log('CLOSING', tagName) - console.log('state0:', state.level, { toolName: (state as any).toolName, paramName: (state as any).paramName }) - - - if (state.level === 'normal') { - fullText += rawNewText - console.log('adding raw A', rawNewText) - } - else if (state.level === 'tool') { - if (tagName === state.toolName) { // closed the tool - state.currentToolCall.isDone = true - state = { - level: 'normal', - } - } - else { // add as text - fullText += rawNewText - console.log('adding raw B', rawNewText) - } - } - else if (state.level === 'param') { - if (tagName === state.paramName) { // closed the param - state.currentToolCall.doneParams.push(state.paramName) - state = { - level: 'tool', - toolName: state.toolName, - currentToolCall: state.currentToolCall, - } - } - else { - fullText += rawNewText - console.log('adding raw C', rawNewText) - - } - } - console.log('state1:', state.level, { toolName: (state as any).toolName, paramName: (state as any).paramName }) - - - }; - - - parser.ontext = (text) => { - if (state.level === 'normal') { - fullText += text - } - // start param - else if (state.level === 'tool') { - // ignore all text in a tool, all text should go in the param tags inside it - } - else if (state.level === 'param') { - if (!(state.paramName in state.currentToolCall.rawParams)) state.currentToolCall.rawParams[state.paramName] = '' - state.currentToolCall.rawParams[state.paramName] += text - } - } + let latestToolCall: RawToolCallObj | undefined = undefined + let foundOpenTag: { idx: number, toolName: ToolName } | null = null + let openToolTagBuffer = '' // the characters we've seen so far that come after a < with no space afterwards, not yet added to fullText let prevFullTextLen = 0 const newOnText: OnText = (params) => { @@ -282,13 +270,55 @@ export const extractToolsWrapper = ( prevFullTextLen = params.fullText.length trueFullText = params.fullText - parser.write(newText) + console.log('NEWTEXT', JSON.stringify(newText)) + + + if (foundOpenTag === null) { + const newFullText = openToolTagBuffer + newText + // ensure the code below doesn't run if only half a tag has been written + const isPartial = findPartiallyWrittenToolTagAtEnd(newFullText, toolOpenTags) + if (isPartial) { + console.log('--- partial!!!') + openToolTagBuffer += newText + } + // if no tooltag is partially written at the end, attempt to get the index + else { + // we will instantly retroactively remove this if it's a tag match + fullText += openToolTagBuffer + openToolTagBuffer = '' + fullText += newText + + const i = findIndexOfAny(fullText, toolOpenTags) + if (i !== null) { + const [idx, toolTag] = i + const toolName = toolTag.substring(1, toolTag.length - 1) as ToolName + console.log('found ', toolName) + foundOpenTag = { idx, toolName } + + // do not count anything at or after i in fullText + fullText = fullText.substring(0, idx) + } + + + } + } + + // toolTagIdx is not null, so parse the XML + if (foundOpenTag !== null) { + latestToolCall = parseXMLPrefixToToolCall( + foundOpenTag.toolName, + trueFullText.substring(foundOpenTag.idx, Infinity), + toolOfToolName, + ) + + } + + - // firstToolCallRef.current === state.currentToolCall is always true onText({ ...params, fullText, - toolCall: firstToolCallRef.current, + toolCall: latestToolCall, }); }; @@ -298,7 +328,7 @@ export const extractToolsWrapper = ( newOnText({ ...params }) fullText = fullText.trimEnd() - const toolCall = firstToolCallRef.current + const toolCall = latestToolCall if (toolCall) { // trim off all whitespace at and before first \n and after last \n for each param for (const p in toolCall.rawParams) { diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/sax.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/sax.ts deleted file mode 100644 index 0d65e943..00000000 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/sax.ts +++ /dev/null @@ -1,150 +0,0 @@ -/*-------------------------------------------------------------------------------------- - * Copyright 2025 Glass Devtools, Inc. All rights reserved. - * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. - *--------------------------------------------------------------------------------------*/ - -// Define options for the parser. -export interface SaxParserOptions { - lowercase?: boolean; -} - -// Define the structure for a parsed node. -export interface SaxNode { - name: string; - attributes: { [key: string]: string }; -} - -// Define the interface for the SAX-like parser. -export interface SaxParser { - // Event handlers that can be set by the consumer. - onopentag: ((node: SaxNode) => void) | null; - ontext: ((text: string) => void) | null; - onclosetag: ((tagName: string) => void) | null; - // Properties to track current positions (used for raw text extraction). - startTagPosition: number; - position: number; - // Processes a new chunk of text. - write(chunk: string): void; -} - -/** - * Creates a minimal, event-driven SAX-like parser. - * - * @param options An object of type `SaxParserOptions`. Passing `{ lowercase: true }` will force all tag names to be lower-cased. - * @returns A parser object implementing the `SaxParser` interface. - */ -export function createSaxParser(options: SaxParserOptions = {}): SaxParser { - // Buffer to hold any leftover text (part of an incomplete tag). - let buffer: string = ''; - // Global counter to track the total processed characters. - let globalPos: number = 0; - - const parser: SaxParser = { - onopentag: null, - ontext: null, - onclosetag: null, - startTagPosition: 0, - position: 0, - - write(chunk: string): void { - // Set the starting position before processing the new chunk. - this.startTagPosition = globalPos; - buffer += chunk; - globalPos += chunk.length; - // Set the current position to the end of the processed chunk. - this.position = globalPos - 1; - - let cursor = 0; - // Flag to indicate if an incomplete tag was found. - let incompleteTagFound = false; - // This will mark the position in the buffer where the incomplete tag starts. - let incompleteStart = 0; - - while (cursor < buffer.length) { - // Look for the next opening '<' character. - const ltIndex = buffer.indexOf('<', cursor); - if (ltIndex === -1) { - // No more tags found in the current buffer. - if (cursor < buffer.length && this.ontext) { - this.ontext(buffer.substring(cursor)); - } - // All content is processed. - buffer = ''; - cursor = buffer.length; - break; - } - - // Emit any text between the current cursor and the opening tag. - if (ltIndex > cursor && this.ontext) { - this.ontext(buffer.substring(cursor, ltIndex)); - } - - // Look for the closing '>' character starting from the found '<'. - const gtIndex = buffer.indexOf('>', ltIndex); - if (gtIndex === -1) { - // Incomplete tag detected. - incompleteTagFound = true; - // Save the starting point of the incomplete tag. - incompleteStart = ltIndex; - break; - } - - // Extract the tag content (excluding the '<' and '>'). - let tagContent = buffer.substring(ltIndex + 1, gtIndex).trim(); - if (!tagContent) { - cursor = gtIndex + 1; - continue; - } - - // Check if this is a closing tag (starts with '/'). - if (tagContent[0] === '/') { - let tagName = tagContent.substring(1).trim(); - if (options.lowercase && tagName) { - tagName = tagName.toLowerCase(); - } - if (this.onclosetag) { - this.onclosetag(tagName); - } - } else { - // Handle self-closing tags (ending with '/'). - let selfClosing = false; - if (tagContent[tagContent.length - 1] === '/') { - selfClosing = true; - tagContent = tagContent.slice(0, -1).trim(); - } - // Determine the tag name (first word before any whitespace). - const spaceIndex = tagContent.indexOf(' '); - let tagName = - spaceIndex !== -1 - ? tagContent.substring(0, spaceIndex).trim() - : tagContent; - if (options.lowercase && tagName) { - tagName = tagName.toLowerCase(); - } - // Emit an open tag event. - if (this.onopentag) { - const node: SaxNode = { name: tagName, attributes: {} }; - this.onopentag(node); - } - // If it’s a self-closing tag, immediately emit a close tag event. - if (selfClosing && this.onclosetag) { - this.onclosetag(tagName); - } - } - // Move the cursor past the current tag. - cursor = gtIndex + 1; - } - - // If an incomplete tag was detected, preserve it. - if (incompleteTagFound) { - // Keep the incomplete portion starting from the '<' - buffer = buffer.substring(incompleteStart); - } else { - // Otherwise, remove all processed content. - buffer = buffer.substring(cursor); - } - }, - }; - - return parser; -}