diff --git a/src/vs/workbench/browser/parts/auxiliarybar/auxiliaryBarPart.ts b/src/vs/workbench/browser/parts/auxiliarybar/auxiliaryBarPart.ts index e940f626..603515a9 100644 --- a/src/vs/workbench/browser/parts/auxiliarybar/auxiliaryBarPart.ts +++ b/src/vs/workbench/browser/parts/auxiliarybar/auxiliaryBarPart.ts @@ -52,7 +52,7 @@ export class AuxiliaryBarPart extends AbstractPaneCompositePart { static readonly viewContainersWorkspaceStateKey = 'workbench.auxiliarybar.viewContainersWorkspaceState'; // Use the side bar dimensions - override readonly minimumWidth: number = 230; // Void changed this (was 170) + override readonly minimumWidth: number = 280; // Void changed this (was 170) override readonly maximumWidth: number = Number.POSITIVE_INFINITY; override readonly minimumHeight: number = 0; override readonly maximumHeight: number = Number.POSITIVE_INFINITY; diff --git a/src/vs/workbench/contrib/void/browser/_dummyContrib.ts b/src/vs/workbench/contrib/void/browser/_dummyContrib.ts index 7f88c445..6898845e 100644 --- a/src/vs/workbench/contrib/void/browser/_dummyContrib.ts +++ b/src/vs/workbench/contrib/void/browser/_dummyContrib.ts @@ -15,7 +15,7 @@ import { KeybindingWeight } from '../../../../platform/keybinding/common/keybind import { IWorkbenchContribution, registerWorkbenchContribution2, WorkbenchPhase } from '../../../common/contributions.js'; -// to change this, just Cmd+Shift+F and replace DummyService with YourServiceName, and replace +// to change this, just Cmd+Shift+F and replace DummyService with YourServiceName, and create a unique ID below export interface IDummyService { readonly _serviceBrand: undefined; // services need this, just leave it undefined } diff --git a/src/vs/workbench/contrib/void/browser/chatThreadService.ts b/src/vs/workbench/contrib/void/browser/chatThreadService.ts index b801a92c..b60563b0 100644 --- a/src/vs/workbench/contrib/void/browser/chatThreadService.ts +++ b/src/vs/workbench/contrib/void/browser/chatThreadService.ts @@ -661,7 +661,6 @@ class ChatThreadService extends Disposable implements IChatThreadService { // above just defines helpers, below starts the actual function const { chatMode } = this._settingsService.state.globalSettings // should not change as we loop even if user changes it, so it goes here - console.log('a', chatMode) // clear any previous error this._setStreamState(threadId, { error: undefined }, 'set') @@ -670,13 +669,11 @@ class ChatThreadService extends Disposable implements IChatThreadService { let isRunningWhenEnd: IsRunningType = undefined let aborted = false - console.log('b') // before enter loop, call tool if (callThisToolFirst) { const { interrupted } = await this._runToolCall(threadId, callThisToolFirst.name, { preapproved: true, validatedParams: callThisToolFirst.params }) if (interrupted) return } - console.log('c') // tool use loop while (shouldSendAnotherMessage) { @@ -688,17 +685,14 @@ class ChatThreadService extends Disposable implements IChatThreadService { let resMessageIsDonePromise: (toolCall?: RawToolCallObj | undefined) => void // resolves when user approves this tool use (or if tool doesn't require approval) const messageIsDonePromise = new Promise((res, rej) => { resMessageIsDonePromise = res }) - console.log('d') // send llm message this._setStreamState(threadId, { isRunning: 'LLM' }, 'merge') const systemMessage = await this._generateSystemMessage(chatMode) - console.log('e0') const llmMessages = await this._generateLLMMessages(threadId) const messages: LLMChatMessage[] = [ { role: 'system', content: systemMessage }, ...llmMessages ] - console.log('e') const llmCancelToken = this._llmMessageService.sendLLMMessage({ messagesType: 'chatMessages', @@ -740,20 +734,14 @@ class ChatThreadService extends Disposable implements IChatThreadService { break } this._setStreamState(threadId, { streamingToken: llmCancelToken }, 'merge') // new stream token for the new message - console.log('waiting...') const toolCall = await messageIsDonePromise // wait for message to complete - console.log('done!') if (aborted) { return } - console.log('H') this._setStreamState(threadId, { streamingToken: undefined }, 'merge') // streaming message is done - console.log('I') // call tool if there is one const tool: RawToolCallObj | undefined = toolCall if (tool) { - console.log('J') const { awaitingUserApproval, interrupted } = await this._runToolCall(threadId, tool.name, { preapproved: false, unvalidatedToolParams: tool.rawParams }) - console.log('K') // stop if interrupted. we don't have to do this for llmMessage because we have a stream token for it and onAbort gets called, but we don't have the equivalent for tools. // just detect tool interruption which is the same as chat interruption right now @@ -768,17 +756,14 @@ class ChatThreadService extends Disposable implements IChatThreadService { } } // end while - console.log('L') // if awaiting user approval, keep isRunning true, else end isRunning this._setStreamState(threadId, { isRunning: isRunningWhenEnd }, 'merge') - console.log('M') // add checkpoint before the next user message if (!isRunningWhenEnd) this._addUserCheckpoint({ threadId }) - console.log('N') // capture number of messages sent this._metricsService.capture('Agent Loop Done', { nMessagesSent, chatMode }) @@ -969,7 +954,7 @@ class ChatThreadService extends Disposable implements IChatThreadService { const [_, toIdx] = c if (toIdx === fromIdx) return - console.log(`going from ${fromIdx} to ${toIdx}`) + // console.log(`going from ${fromIdx} to ${toIdx}`) // update the user's checkpoint this._addUserModificationsToCurrCheckpoint({ threadId }) @@ -1064,6 +1049,7 @@ We only need to do it for files that were edited since `from`, ie files between severity: error ? Severity.Warning : Severity.Info, message: error ? `Error: ${error} ` : `A new Chat result is ready.`, source: messageContent, + sticky: true, actions: { primary: [{ id: 'void.goToChat', @@ -1500,9 +1486,9 @@ We only need to do it for files that were edited since `from`, ie files between } }, true) - // when change focused message idx, jump - if (messageIdx !== undefined) - this.jumpToCheckpointBeforeMessageIdx({ threadId, messageIdx, jumpToUserModified: true }) + // // when change focused message idx, jump - do not jump back when click edit, too confusing. + // if (messageIdx !== undefined) + // this.jumpToCheckpointBeforeMessageIdx({ threadId, messageIdx, jumpToUserModified: true }) } // set message.state diff --git a/src/vs/workbench/contrib/void/browser/directoryStrService.ts b/src/vs/workbench/contrib/void/browser/directoryStrService.ts index 09fc4f04..ff5972bc 100644 --- a/src/vs/workbench/contrib/void/browser/directoryStrService.ts +++ b/src/vs/workbench/contrib/void/browser/directoryStrService.ts @@ -14,18 +14,23 @@ import { MAX_CHILDREN_URIs_PAGE } from './toolsService.js'; import { IExplorerService } from '../../files/browser/files.js'; import { SortOrder } from '../../files/common/files.js'; import { ExplorerItem } from '../../files/common/explorerModel.js'; -import { VoidDirectoryItem } from '../common/directoryStrTypes.js'; import { MAX_DIRSTR_CHARS_TOTAL_BEGINNING, MAX_DIRSTR_CHARS_TOTAL_TOOL } from '../common/prompt/prompts.js'; -// const MAX_FILES_TOTAL = 200 +const MAX_FILES_TOTAL = 300; + +const DEFAULT_MAX_DEPTH = 3; +const DEFAULT_MAX_ITEMS_PER_DIR = 3; + +const START_MAX_DEPTH = Infinity; +const START_MAX_ITEMS_PER_DIR = Infinity; // Add start value as Infinity export interface IDirectoryStrService { readonly _serviceBrand: undefined; - getDirectoryStrTool(uri: URI): Promise - getAllDirectoriesStr(opts: { cutOffMessage: string }): Promise + getDirectoryStrTool(uri: URI, options?: { maxItemsPerDir?: number }): Promise + getAllDirectoriesStr(opts: { cutOffMessage: string, maxItemsPerDir?: number }): Promise } export const IDirectoryStrService = createDecorator('voidDirectoryStrService'); @@ -53,11 +58,17 @@ const shouldExcludeDirectory = (item: ExplorerItem) => { item.name === 'obj' || item.name === 'vendor' || item.name === 'logs' || - item.name === 'cache' + item.name === 'cache' || + item.name === 'resource' || + item.name === 'resources' ) { return true; } + + if (item.name.match(/\bout\b/)) return true + if (item.name.match(/\bbuild\b/)) return true + return false; } @@ -128,137 +139,175 @@ export const stringifyDirectoryTree1Deep = (params: ToolCallParams['ls_dir'], re // ---------- IN GENERAL ---------- - -// if the filter exists use it to filter out files and folders when creating the tree -const computeDirectoryTree = async ( +// Remove the old computeDirectoryTree function and replace with a combined version that handles both computation and rendering +const computeAndStringifyDirectoryTree = async ( eItem: ExplorerItem, - explorerService: IExplorerService -): Promise => { - // Fetch children with default sort order - console.log('11111!!!!') - const eChildren = await eItem.fetchChildren(SortOrder.FilesFirst); - console.log('222222!!!!') - - const isGitIgnoredDirectory = eItem.isDirectory && shouldExcludeDirectory(eItem) - - // Process children recursively - const children = !isGitIgnoredDirectory ? await Promise.all( - eChildren.map(async c => await computeDirectoryTree(c, explorerService)) - ) : null - console.log('333333!!!!!') - - - // Create our directory item - const item: VoidDirectoryItem = { - uri: eItem.resource, - name: eItem.name, - isDirectory: eItem.isDirectory, - isSymbolicLink: eItem.isSymbolicLink, - children, - isGitIgnoredDirectory: isGitIgnoredDirectory && { numChildren: eItem.children.size }, - }; - - return item; -}; - - -const stringifyDirectoryTree = ( - node: VoidDirectoryItem, + explorerService: IExplorerService, MAX_CHARS: number, -): { content: string, wasCutOff: boolean } => { - let content = ''; - let wasCutOff = false; + fileCount: { count: number } = { count: 0 }, + options: { maxDepth?: number, currentDepth?: number, maxItemsPerDir?: number } = {} +): Promise<{ content: string, wasCutOff: boolean }> => { + // Set default values for options + const maxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH; + const currentDepth = options.currentDepth ?? 0; + const maxItemsPerDir = options.maxItemsPerDir ?? DEFAULT_MAX_ITEMS_PER_DIR; + + // Check if we've reached the max depth + if (currentDepth > maxDepth) { + return { content: '', wasCutOff: true }; + } + + // Check if we've reached the file limit + if (fileCount.count >= MAX_FILES_TOTAL) { + return { content: '', wasCutOff: true }; + } // If we're already exceeding the max characters, return immediately if (MAX_CHARS <= 0) { - return { content, wasCutOff: true }; + return { content: '', wasCutOff: true }; } + // Increment file count + fileCount.count++; + // Add the root node first (without tree characters) - const nodeLine = `${node.name}${node.isDirectory ? '/' : ''}${node.isSymbolicLink ? ' (symbolic link)' : ''}\n`; + const nodeLine = `${eItem.name}${eItem.isDirectory ? '/' : ''}${eItem.isSymbolicLink ? ' (symbolic link)' : ''}\n`; if (nodeLine.length > MAX_CHARS) { return { content: '', wasCutOff: true }; } - content += nodeLine; + let content = nodeLine; + let wasCutOff = false; let remainingChars = MAX_CHARS - nodeLine.length; - // Then recursively add all children with proper tree formatting - if (node.children && node.children.length > 0) { - const { childrenContent, childrenCutOff } = renderChildren( - node.children, - remainingChars, - '' - ); - content += childrenContent; - wasCutOff = childrenCutOff; + // Check if it's a directory we should skip + const isGitIgnoredDirectory = eItem.isDirectory && shouldExcludeDirectory(eItem); + + // Fetch and process children if not a filtered directory + if (eItem.isDirectory && !isGitIgnoredDirectory) { + // Fetch children with Modified sort order to show recently modified first + const eChildren = await eItem.fetchChildren(SortOrder.Modified); + + // Then recursively add all children with proper tree formatting + if (eChildren && eChildren.length > 0) { + const { childrenContent, childrenCutOff } = await renderChildrenCombined( + eChildren, + remainingChars, + '', + explorerService, + fileCount, + { maxDepth, currentDepth, maxItemsPerDir } // Pass maxItemsPerDir to the render function + ); + content += childrenContent; + wasCutOff = childrenCutOff; + } } + return { content, wasCutOff }; }; // Helper function to render children with proper tree formatting -const renderChildren = ( - children: VoidDirectoryItem[], +const renderChildrenCombined = async ( + children: ExplorerItem[], maxChars: number, - parentPrefix: string -): { childrenContent: string, childrenCutOff: boolean } => { + parentPrefix: string, + explorerService: IExplorerService, + fileCount: { count: number }, + options: { maxDepth: number, currentDepth: number, maxItemsPerDir?: number } +): Promise<{ childrenContent: string, childrenCutOff: boolean }> => { + const { maxDepth, currentDepth } = options; // Remove maxItemsPerDir from destructuring + // Get maxItemsPerDir separately and make sure we use it + // For first level (currentDepth = 0), always use Infinity regardless of what was passed + const maxItemsPerDir = currentDepth === 0 ? + Infinity : + (options.maxItemsPerDir ?? DEFAULT_MAX_ITEMS_PER_DIR); + const nextDepth = currentDepth + 1; + let childrenContent = ''; let childrenCutOff = false; + let remainingChars = maxChars; - for (let i = 0; i < children.length; i++) { - const child = children[i]; - const isLast = i === children.length - 1; + // Check if we've reached max depth + if (nextDepth > maxDepth) { + return { childrenContent: '', childrenCutOff: true }; + } - console.log('child!!!!', child.uri.fsPath) + // Apply maxItemsPerDir limit - only process the specified number of items + const itemsToProcess = maxItemsPerDir === Infinity ? children : children.slice(0, maxItemsPerDir); + const hasMoreItems = children.length > itemsToProcess.length; + for (let i = 0; i < itemsToProcess.length; i++) { + // Check if we've reached the file limit + if (fileCount.count >= MAX_FILES_TOTAL) { + childrenCutOff = true; + break; + } + + const child = itemsToProcess[i]; + const isLast = (i === itemsToProcess.length - 1) && !hasMoreItems; // Create the tree branch symbols const branchSymbol = isLast ? '└── ' : '├── '; const childLine = `${parentPrefix}${branchSymbol}${child.name}${child.isDirectory ? '/' : ''}${child.isSymbolicLink ? ' (symbolic link)' : ''}\n`; // Check if adding this line would exceed the limit - if (childrenContent.length + childLine.length > maxChars) { + if (childLine.length > remainingChars) { childrenCutOff = true; break; } + childrenContent += childLine; + remainingChars -= childLine.length; + fileCount.count++; const nextLevelPrefix = parentPrefix + (isLast ? ' ' : '│ '); - - // if gitignored, just say the number of children - if (child.isDirectory && child.isGitIgnoredDirectory && child.isGitIgnoredDirectory.numChildren > 0) { - childrenContent += `${nextLevelPrefix}└── ... (${child.isGitIgnoredDirectory.numChildren} children) ...\n` - } + // Skip processing children for git ignored directories + const isGitIgnoredDirectory = child.isDirectory && shouldExcludeDirectory(child); // Create the prefix for the next level (continuation line or space) - else if (child.children && child.children.length > 0) { + if (child.isDirectory && !isGitIgnoredDirectory) { + // Fetch children with Modified sort order to show recently modified first + const eChildren = await child.fetchChildren(SortOrder.Modified); - const { - childrenContent: grandChildrenContent, - childrenCutOff: grandChildrenCutOff - } = renderChildren( - child.children, - maxChars, - nextLevelPrefix - ); + if (eChildren && eChildren.length > 0) { + const { + childrenContent: grandChildrenContent, + childrenCutOff: grandChildrenCutOff + } = await renderChildrenCombined( + eChildren, + remainingChars, + nextLevelPrefix, + explorerService, + fileCount, + { maxDepth, currentDepth: nextDepth, maxItemsPerDir } + ); - // If adding grandchildren content would exceed the limit - if (childrenContent.length + grandChildrenContent.length > maxChars) { - childrenCutOff = true; - break; - } + if (grandChildrenContent.length > 0) { + childrenContent += grandChildrenContent; + remainingChars -= grandChildrenContent.length; + } - childrenContent += grandChildrenContent; - - if (grandChildrenCutOff) { - childrenCutOff = true; - break; + if (grandChildrenCutOff) { + childrenCutOff = true; + } } } } + // Add a message if we truncated the items due to maxItemsPerDir + if (hasMoreItems) { + const remainingCount = children.length - itemsToProcess.length; + const truncatedLine = `${parentPrefix}└── (${remainingCount} more items not shown...)\n`; + + if (truncatedLine.length <= remainingChars) { + childrenContent += truncatedLine; + remainingChars -= truncatedLine.length; + } + childrenCutOff = true; + } + return { childrenContent, childrenCutOff }; }; @@ -276,12 +325,37 @@ class DirectoryStrService extends Disposable implements IDirectoryStrService { super(); } - async getDirectoryStrTool(uri: URI) { + async getDirectoryStrTool(uri: URI, options?: { maxItemsPerDir?: number }) { const eRoot = this.explorerService.findClosest(uri) if (!eRoot) throw new Error(`There was a problem reading the URI: ${uri.fsPath}.`) - const dirTree = await computeDirectoryTree(eRoot, this.explorerService); - const { content, wasCutOff } = stringifyDirectoryTree(dirTree, MAX_DIRSTR_CHARS_TOTAL_TOOL); + const maxItemsPerDir = options?.maxItemsPerDir ?? START_MAX_ITEMS_PER_DIR; // Use START_MAX_ITEMS_PER_DIR + + // First try with START_MAX_DEPTH + const { content: initialContent, wasCutOff: initialCutOff } = await computeAndStringifyDirectoryTree( + eRoot, + this.explorerService, + MAX_DIRSTR_CHARS_TOTAL_TOOL, + { count: 0 }, + { maxDepth: START_MAX_DEPTH, currentDepth: 0, maxItemsPerDir } + ); + + // If cut off, try again with DEFAULT_MAX_DEPTH and DEFAULT_MAX_ITEMS_PER_DIR + let content, wasCutOff; + if (initialCutOff) { + const result = await computeAndStringifyDirectoryTree( + eRoot, + this.explorerService, + MAX_DIRSTR_CHARS_TOTAL_TOOL, + { count: 0 }, + { maxDepth: DEFAULT_MAX_DEPTH, currentDepth: 0, maxItemsPerDir: DEFAULT_MAX_ITEMS_PER_DIR } + ); + content = result.content; + wasCutOff = result.wasCutOff; + } else { + content = initialContent; + wasCutOff = initialCutOff; + } let c = content.substring(0, MAX_DIRSTR_CHARS_TOTAL_TOOL) c = `Directory of ${uri.fsPath}:\n${content}` @@ -290,13 +364,16 @@ class DirectoryStrService extends Disposable implements IDirectoryStrService { return c } - async getAllDirectoriesStr({ cutOffMessage }: { cutOffMessage: string }) { + async getAllDirectoriesStr({ cutOffMessage, maxItemsPerDir }: { cutOffMessage: string, maxItemsPerDir?: number }) { let str: string = ''; let cutOff = false; const folders = this.workspaceContextService.getWorkspace().folders; if (folders.length === 0) return '(NO WORKSPACE OPEN)'; + // Use START_MAX_ITEMS_PER_DIR if not specified + const startMaxItemsPerDir = maxItemsPerDir ?? START_MAX_ITEMS_PER_DIR; + for (let i = 0; i < folders.length; i += 1) { if (i > 0) str += '\n'; @@ -308,19 +385,44 @@ class DirectoryStrService extends Disposable implements IDirectoryStrService { const eRoot = this.explorerService.findClosestRoot(rootURI); if (!eRoot) continue; - // Use our new approach with direct explorer service - const dirTree = await computeDirectoryTree(eRoot, this.explorerService); - const { content, wasCutOff } = stringifyDirectoryTree(dirTree, MAX_DIRSTR_CHARS_TOTAL_BEGINNING - str.length); + // First try with START_MAX_DEPTH and startMaxItemsPerDir + const { content: initialContent, wasCutOff: initialCutOff } = await computeAndStringifyDirectoryTree( + eRoot, + this.explorerService, + MAX_DIRSTR_CHARS_TOTAL_BEGINNING - str.length, + { count: 0 }, + { maxDepth: START_MAX_DEPTH, currentDepth: 0, maxItemsPerDir: startMaxItemsPerDir } + ); + + // If cut off, try again with DEFAULT_MAX_DEPTH and DEFAULT_MAX_ITEMS_PER_DIR + let content, wasCutOff; + if (initialCutOff) { + const result = await computeAndStringifyDirectoryTree( + eRoot, + this.explorerService, + MAX_DIRSTR_CHARS_TOTAL_BEGINNING - str.length, + { count: 0 }, + { maxDepth: DEFAULT_MAX_DEPTH, currentDepth: 0, maxItemsPerDir: DEFAULT_MAX_ITEMS_PER_DIR } + ); + content = result.content; + wasCutOff = result.wasCutOff; + } else { + content = initialContent; + wasCutOff = initialCutOff; + } + str += content; if (wasCutOff) { cutOff = true; break; } } + console.log('cutoff!!!!!!!', str, cutOffMessage) if (cutOff) { return `${str}\n${cutOffMessage}` } + return str } } diff --git a/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx b/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx index bad585f8..4ef01353 100644 --- a/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx +++ b/src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx @@ -22,12 +22,13 @@ import { WarningBox } from '../void-settings-tsx/WarningBox.js'; import { getModelCapabilities, getIsReasoningEnabledState } from '../../../../common/modelCapabilities.js'; import { AlertTriangle, Ban, Check, ChevronRight, Dot, FileIcon, Pencil, Undo, Undo2, X } from 'lucide-react'; import { ChatMessage, CheckpointEntry, StagingSelectionItem, ToolMessage } from '../../../../common/chatThreadServiceTypes.js'; -import { ToolCallParams, ToolNameWithApproval } from '../../../../common/toolsServiceTypes.js'; +import { LintErrorItem, ToolCallParams, ToolNameWithApproval } from '../../../../common/toolsServiceTypes.js'; import { ApplyButtonsHTML, CopyButton, IconShell1, JumpToFileButton, JumpToTerminalButton, StatusIndicator, StatusIndicatorForApplyButton, useApplyButtonState } from '../markdown/ApplyBlockHoverButtons.js'; import { IsRunningType } from '../../../chatThreadService.js'; import { acceptAllBg, acceptBorder, buttonFontSize, buttonTextColor, rejectAllBg, rejectBg, rejectBorder } from '../../../../common/helpers/colors.js'; import { PlacesType } from 'react-tooltip'; import { ToolName, toolNames } from '../../../../common/prompt/prompts.js'; +import { error } from 'console'; @@ -671,6 +672,7 @@ const ToolHeaderWrapper = ({ numResults, hasNextPage, children, + bottomChildren, isError, onClick, isOpen, @@ -733,6 +735,7 @@ const ToolHeaderWrapper = ({ {children} } + {bottomChildren} ); }; @@ -1338,6 +1341,28 @@ const EditToolChildren = ({ uri, changeDescription }: { uri: URI, changeDescript } +const EditToolLintErrors = ({ lintErrors }: { lintErrors: LintErrorItem[] }) => { + + if (lintErrors.length === 0) return null; + + return ( +
+
+ +
+ {lintErrors.map((error, i) => ( +
Lines {error.startLineNumber}-{error.endLineNumber}: {error.message}
+ ))} +
+ +
+
+ ) + + +} + + const EditToolHeaderButtons = ({ applyBoxId, uri, codeStr }: { applyBoxId: string, uri: URI, codeStr: string }) => { const { currStreamState } = useApplyButtonState({ applyBoxId, uri }) return
@@ -1726,7 +1751,7 @@ const toolNameToComponent: { [T in ToolName]: { resultWrapper: ResultWrapper, if (toolMessage.type !== 'tool_error') { const { params, result } = toolMessage - // componentParams.bottomChildren = + componentParams.bottomChildren = componentParams.children = (null) - const [userChosenProviderName, setUserChosenProviderName] = useState(null) + const [userChosenProviderName, setUserChosenProviderName] = useState('anthropic') const providerName = permanentProviderName ?? userChosenProviderName; const [modelName, setModelName] = useState('') const [errorString, setErrorString] = useState('') - if (!providerName) { return null; } - const numModels = settingsState.settingsOfProvider[providerName].models.length if (!isOpen) { @@ -206,9 +204,8 @@ const AddModelInputBox = ({ providerName: permanentProviderName, className, comp getOptionDisplayName={(pn) => pn ? displayInfoOfProviderName(pn).title : 'Provider Name'} getOptionDropdownName={(pn) => pn ? displayInfoOfProviderName(pn).title : 'Provider Name'} getOptionsEqual={(a, b) => a === b} - className={`max-w-44 w-full border border-void-border-2 bg-void-bg-1 text-void-fg-3 text-root - py-[4px] px-[6px] - `} + // className={`max-w-44 w-full border border-void-border-2 bg-void-bg-1 text-void-fg-3 text-root py-[4px] px-[6px]`} + className={`max-w-32 mx-2 w-full resize-none bg-void-bg-1 text-void-fg-1 placeholder:text-void-fg-3 border border-void-border-2 focus:border-void-border-1 py-1 px-2 rounded`} arrowTouchesText={false} /> } @@ -219,7 +216,7 @@ const AddModelInputBox = ({ providerName: permanentProviderName, className, comp onChangeValue={setModelName} placeholder='Model Name' compact={compact} - className={'max-w-44'} + className={'max-w-32'} /> {/* add button */} @@ -299,7 +296,14 @@ export const ModelDump = () => { {modelName}
{/* right part is anything that fits */} -
+
{isAutodetected ? '(detected locally)' : isDefault ? '' : '(custom model)'} {

Models

- + @@ -641,6 +645,16 @@ export const FeaturesTab = () => { /> {voidSettingsState.globalSettings.autoApprove ? 'Auto-approve' : 'Auto-approve'}
+ + {/* Tool Lint Errors Switch */} +
+ voidSettingsService.setGlobalSetting('includeToolLintErrors', newVal)} + /> + {voidSettingsState.globalSettings.includeToolLintErrors ? 'Include after-edit lint errors' : `Don't include lint errors`} +
@@ -890,7 +904,7 @@ const GeneralTab = () => { return <>

One-Click Switch

-

{`Transfer your settings from another editor to Void in one click.`}

+

{`Transfer your settings from another editor to Void in one click.`}

@@ -903,7 +917,7 @@ const GeneralTab = () => {

Built-in Settings

-

{`IDE settings, keyboard settings, and theme customization.`}

+

{`IDE settings, keyboard settings, and theme customization.`}

{ commandService.executeCommand('workbench.action.openSettings') }}> @@ -930,7 +944,7 @@ const GeneralTab = () => {

AI Instructions

-

{`Instructions to include on all AI requests.`}

+

{`Instructions to include on all AI requests.`}

@@ -946,7 +960,7 @@ export const Settings = () => { const [tab, setTab] = useState('models') - const deleteme = false + const deleteme = true if (deleteme) { return
@@ -1103,7 +1117,7 @@ const PreviousButton = ({ onClick, ...props }: { onClick: () => void } & React.B } -const ollamaSetupInstructions =
+const ollamaSetupInstructions =
@@ -1440,38 +1454,44 @@ const VoidOnboarding = () => { -
AI Preferences
+
AI Preferences
What are you looking for in an AI model?
-
+
{ setWantToUseOption('smart'); setPageIndex(pageIndex + 1); }} - className="flex flex-col items-center justify-center p-6 rounded-md transition-all duration-300 cursor-pointer md:aspect-[8/7] border-void-border-1 border bg-gradient-to-br from-[#0e70c0]/15 via-[#0e70c0]/5 to-transparent hover:from-[#0e70c0]/25 hover:via-[#0e70c0]/10 hover:to-[#0e70c0]/5 dark:from-[#0e70c0]/20 dark:via-[#0e70c0]/10 dark:to-[#0e70c0]/5 dark:hover:from-[#0e70c0]/30 dark:hover:via-[#0e70c0]/15 dark:hover:to-[#0e70c0]/5" + className="flex flex-col items-center w-full justify-center p-6 rounded-md cursor-pointer md:aspect-[8/7] border-void-border-1 border relative overflow-hidden group" > - 🧠 -

Intelligence

-

{basicDescOfWantToUseOption['smart']}

+
+
+ 🧠 +

Intelligence

+

{basicDescOfWantToUseOption['smart']}

{ setWantToUseOption('private'); setPageIndex(pageIndex + 1); }} - className="flex flex-col items-center justify-center p-6 rounded-md transition-all duration-300 cursor-pointer md:aspect-[8/7] border-void-border-1 border bg-gradient-to-br from-[#0e70c0]/15 via-[#0e70c0]/5 to-transparent hover:from-[#0e70c0]/25 hover:via-[#0e70c0]/10 hover:to-[#0e70c0]/5 dark:from-[#0e70c0]/20 dark:via-[#0e70c0]/10 dark:to-[#0e70c0]/5 dark:hover:from-[#0e70c0]/30 dark:hover:via-[#0e70c0]/15 dark:hover:to-[#0e70c0]/5" + className="flex flex-col items-center w-full justify-center p-6 rounded-md cursor-pointer md:aspect-[8/7] border-void-border-1 border relative overflow-hidden group" > - 🔒 -

Privacy

-

{basicDescOfWantToUseOption['private']}

+
+
+ 🔒 +

Privacy

+

{basicDescOfWantToUseOption['private']}

{ setWantToUseOption('cheap'); setPageIndex(pageIndex + 1); }} - className="flex flex-col items-center justify-center p-6 rounded-md transition-all duration-300 cursor-pointer md:aspect-[8/7] border-void-border-1 border bg-gradient-to-br from-[#0e70c0]/15 via-[#0e70c0]/5 to-transparent hover:from-[#0e70c0]/25 hover:via-[#0e70c0]/10 hover:to-[#0e70c0]/5 dark:from-[#0e70c0]/20 dark:via-[#0e70c0]/10 dark:to-[#0e70c0]/5 dark:hover:from-[#0e70c0]/30 dark:hover:via-[#0e70c0]/15 dark:hover:to-[#0e70c0]/5" + className="flex flex-col items-center w-full justify-center p-6 rounded-md cursor-pointer md:aspect-[8/7] border-void-border-1 border relative overflow-hidden group" > - 💵 -

Low-Cost

-

{basicDescOfWantToUseOption['cheap']}

+
+
+ 💵 +

Low-Cost

+

{basicDescOfWantToUseOption['cheap']}

@@ -1630,11 +1650,11 @@ const VoidOnboarding = () => {
Settings and Themes
-
-

Transfer your settings from an existing editor?

- - - +
+

Transfer your settings from an existing editor?

+ + +
@@ -1655,7 +1675,7 @@ const VoidOnboarding = () => { } - return
+ return
{contentOfIdx[pageIndex]}
} diff --git a/src/vs/workbench/contrib/void/browser/toolsService.ts b/src/vs/workbench/contrib/void/browser/toolsService.ts index 83249778..8dec053d 100644 --- a/src/vs/workbench/contrib/void/browser/toolsService.ts +++ b/src/vs/workbench/contrib/void/browser/toolsService.ts @@ -18,6 +18,7 @@ import { IMarkerService } from '../../../../platform/markers/common/markers.js' import { timeout } from '../../../../base/common/async.js' import { RawToolParamsObj } from '../common/sendLLMMessageTypes.js' import { ToolName } from '../common/prompt/prompts.js' +import { IVoidSettingsService } from '../common/voidSettingsService.js' // tool use for AI @@ -151,6 +152,7 @@ export class ToolsService implements IToolsService { @IVoidCommandBarService private readonly commandBarService: IVoidCommandBarService, @IDirectoryStrService private readonly directoryStrService: IDirectoryStrService, @IMarkerService private readonly markerService: IMarkerService, + @IVoidSettingsService private readonly voidSettingsService: IVoidSettingsService, ) { const queryBuilder = instantiationService.createInstance(QueryBuilder); @@ -412,10 +414,13 @@ export class ToolsService implements IToolsService { return `URI ${params.uri.fsPath} successfully deleted.` }, edit_file: (params, result) => { + const lintErrsString = ( + this.voidSettingsService.state.globalSettings.includeToolLintErrors ? + (result.lintErrors ? ` Lint errors found after change:\n${lintErrorsStr(result.lintErrors)}.\nIf this is related to a change made while calling this tool, you might want to fix the error.` + : ` No lint errors found.`) + : '') - const additionalStr = result.lintErrors ? `Lint errors found after change:\n${lintErrorsStr(result.lintErrors)}.\nIf this is related to a change made while calling this tool, you might want to fix the error.` : `No lint errors found.` - - return `Change successfully made to ${params.uri.fsPath}.${additionalStr}` + return `Change successfully made to ${params.uri.fsPath}.${lintErrsString}` }, run_terminal_command: (params, result) => { const { diff --git a/src/vs/workbench/contrib/void/common/helpers/extractCodeFromResult.ts b/src/vs/workbench/contrib/void/common/helpers/extractCodeFromResult.ts index 3a131461..09e57f04 100644 --- a/src/vs/workbench/contrib/void/common/helpers/extractCodeFromResult.ts +++ b/src/vs/workbench/contrib/void/common/helpers/extractCodeFromResult.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------*/ import { DIVIDER, FINAL, ORIGINAL } from '../prompt/prompts.js' -class SurroundingsRemover { +export class SurroundingsRemover { readonly originalS: string i: number j: number @@ -58,12 +58,13 @@ class SurroundingsRemover { // return offset === suffix.length // } + // either removes all or nothing removeFromStartUntilFullMatch = (until: string, alsoRemoveUntilStr: boolean) => { const index = this.originalS.indexOf(until, this.i) if (index === -1) { - this.i = this.j + 1 - return null + // this.i = this.j + 1 + return false } // console.log('index', index, until.length) diff --git a/src/vs/workbench/contrib/void/common/prompt/prompts.ts b/src/vs/workbench/contrib/void/common/prompt/prompts.ts index 1301cdbc..100c01c0 100644 --- a/src/vs/workbench/contrib/void/common/prompt/prompts.ts +++ b/src/vs/workbench/contrib/void/common/prompt/prompts.ts @@ -16,6 +16,10 @@ export const tripleTick = ['```', '```'] export const MAX_DIRSTR_CHARS_TOTAL_BEGINNING = 20_000 export const MAX_DIRSTR_CHARS_TOTAL_TOOL = 20_000 +export const MAX_DIRSTR_RESULTS_TOTAL_BEGINNING = 100 +export const MAX_DIRSTR_RESULTS_TOTAL_TOOL = 100 + + export const MAX_PREFIX_SUFFIX_CHARS = 20_000 @@ -300,8 +304,8 @@ ${directoryStr} } if (mode === 'gather') { - details.push(`Your primary use of tools should be to gather information to help the user understand the codebase and answer their query.`) - details.push(`You should extensively read files, types, content, etc and gather relevant context.`) + details.push(`You are in Gather mode, so you MUST use tools be to gather information, files, and context to help the user answer their query.`) + details.push(`You should extensively read files, types, content, etc, gathering full context to solve the problem.`) } diff --git a/src/vs/workbench/contrib/void/common/voidSettingsService.ts b/src/vs/workbench/contrib/void/common/voidSettingsService.ts index b4ddb7ab..ad237922 100644 --- a/src/vs/workbench/contrib/void/common/voidSettingsService.ts +++ b/src/vs/workbench/contrib/void/common/voidSettingsService.ts @@ -211,7 +211,15 @@ class VoidSettingsService extends Disposable implements IVoidSettingsService { } async readAndInitializeState() { - const readS = await this._readState(); + let readS: VoidSettingsState + try { + readS = await this._readState(); + // 1.0.3 addition, remove when enough users have had this code run + if (readS.globalSettings.includeToolLintErrors === undefined) readS.globalSettings.includeToolLintErrors = true + } + catch (e) { + readS = defaultState() + } // the stored data structure might be outdated, so we need to update it here const finalState = readS diff --git a/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts b/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts index 47c1f515..bf54f61a 100644 --- a/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts +++ b/src/vs/workbench/contrib/void/common/voidSettingsTypes.ts @@ -356,6 +356,7 @@ export type GlobalSettings = { chatMode: ChatMode; autoApprove: boolean; showInlineSuggestions: boolean; + includeToolLintErrors: boolean; } export const defaultGlobalSettings: GlobalSettings = { @@ -367,6 +368,7 @@ export const defaultGlobalSettings: GlobalSettings = { chatMode: 'agent', autoApprove: false, showInlineSuggestions: true, + includeToolLintErrors: true, } export type GlobalSettingName = keyof GlobalSettings diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/extractGrammar.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/extractGrammar.ts index 0fbb1b29..463d0c06 100644 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/extractGrammar.ts +++ b/src/vs/workbench/contrib/void/electron-main/llmMessage/extractGrammar.ts @@ -3,11 +3,10 @@ * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. *--------------------------------------------------------------------------------------*/ -import { endsWithAnyPrefixOf } from '../../common/helpers/extractCodeFromResult.js' +import { endsWithAnyPrefixOf, SurroundingsRemover } from '../../common/helpers/extractCodeFromResult.js' import { availableTools, InternalToolInfo, ToolName, ToolParamName } from '../../common/prompt/prompts.js' -import { OnFinalMessage, OnText, RawToolCallObj } from '../../common/sendLLMMessageTypes.js' +import { OnFinalMessage, OnText, RawToolCallObj, RawToolParamsObj } from '../../common/sendLLMMessageTypes.js' import { ChatMode } from '../../common/voidSettingsTypes.js' -import { createSaxParser } from './sax.js' // =============== reasoning =============== @@ -137,17 +136,122 @@ export const extractReasoningWrapper = ( // =============== tools =============== -type ToolsState = { - level: 'normal', -} | { - level: 'tool', - toolName: ToolName, - currentToolCall: RawToolCallObj, -} | { - level: 'param', - toolName: ToolName, - paramName: ToolParamName, - currentToolCall: RawToolCallObj, + + +const findPartiallyWrittenToolTagAtEnd = (fullText: string, toolTags: string[]) => { + for (const toolTag of toolTags) { + const foundPrefix = endsWithAnyPrefixOf(fullText, toolTag) + if (foundPrefix) { + return [foundPrefix, toolTag] as const + } + } + return false +} + +const findIndexOfAny = (fullText: string, matches: string[]) => { + for (const str of matches) { + const idx = fullText.indexOf(str); + if (idx !== -1) { + return [idx, str] as const + } + } + return null +} + + +type ToolOfToolName = { [toolName: string]: InternalToolInfo | undefined } +const parseXMLPrefixToToolCall = (toolName: ToolName, str: string, toolOfToolName: ToolOfToolName): RawToolCallObj => { + const paramsObj: RawToolParamsObj = {} + const doneParams: ToolParamName[] = [] + let isDone = false + + const getAnswer = (): RawToolCallObj => { + // trim off all whitespace at and before first \n and after last \n for each param + for (const p in paramsObj) { + const paramName = p as ToolParamName + const orig = paramsObj[paramName] + if (orig === undefined) continue + paramsObj[paramName] = trimBeforeAndAfterNewLines(orig) + } + + // return tool call + const ans: RawToolCallObj = { + name: toolName, + rawParams: paramsObj, + doneParams: doneParams, + isDone: isDone + } + return ans + } + + // find first toolName tag + const openToolTag = `<${toolName}>` + let i = str.indexOf(openToolTag) + if (i === -1) return getAnswer() + let j = str.lastIndexOf(``) + if (j === -1) j = Infinity + else isDone = true + + + str = str.substring(i + openToolTag.length, j) + + const pm = new SurroundingsRemover(str) + + const allowedParams = Object.keys(toolOfToolName[toolName]?.params ?? {}) as ToolParamName[] + if (allowedParams.length === 0) return getAnswer() + let latestMatchedOpenParam: null | ToolParamName = null + let n = 0 + while (true) { + n += 1 + if (n > 10) return getAnswer() // just for good measure as this code is early + + // find the param name opening tag + let matchedOpenParam: null | ToolParamName = null + for (const paramName of allowedParams) { + const removed = pm.removeFromStartUntilFullMatch(`<${paramName}>`, true) + if (removed) { + matchedOpenParam = paramName + break + } + } + // if did not find a new param, stop + if (matchedOpenParam === null) { + if (latestMatchedOpenParam !== null) { + paramsObj[latestMatchedOpenParam] += pm.value() + } + return getAnswer() + } + else { + latestMatchedOpenParam = matchedOpenParam + } + + paramsObj[latestMatchedOpenParam] = '' + + // find the param name closing tag + let matchedCloseParam: boolean = false + let paramContents = '' + for (const paramName of allowedParams) { + const i = pm.i + const closeTag = `` + const removed = pm.removeFromStartUntilFullMatch(closeTag, true) + if (removed) { + const i2 = pm.i + paramContents = pm.originalS.substring(i, i2 - closeTag.length) + matchedCloseParam = true + break + } + } + // if did not find a new close tag, stop + if (!matchedCloseParam) { + paramsObj[latestMatchedOpenParam] += pm.value() + return getAnswer() + } + else { + doneParams.push(latestMatchedOpenParam) + } + + paramsObj[latestMatchedOpenParam] += paramContents + } } export const extractToolsWrapper = ( @@ -156,125 +260,17 @@ export const extractToolsWrapper = ( const tools = availableTools(chatMode) if (!tools) return { newOnText: onText, newOnFinalMessage: onFinalMessage } - const toolOfToolName: { [toolName: string]: InternalToolInfo | undefined } = {} + const toolOfToolName: ToolOfToolName = {} + const toolOpenTags = tools.map(t => `<${t.name}>`) for (const t of tools) { toolOfToolName[t.name] = t } // detect , etc let fullText = ''; let trueFullText = '' - const firstToolCallRef: { current: RawToolCallObj | undefined } = { current: undefined } - - let state: ToolsState = { level: 'normal' } - - - const getRawNewText = () => { - return trueFullText.substring(parser.startTagPosition, parser.position + 1) - } - const parser = createSaxParser() - - // when see open tag - parser.onopentag = (node) => { - const rawNewText = getRawNewText() - const tagName = node.name; - console.log('OPENING', tagName) - console.log('state0:', state.level, { toolName: (state as any).toolName, paramName: (state as any).paramName }) - - if (state.level === 'normal') { - if (tagName in toolOfToolName) { // valid toolName - state = { - level: 'tool', - toolName: tagName as ToolName, - currentToolCall: { name: tagName as ToolName, rawParams: {}, doneParams: [], isDone: false } - } - firstToolCallRef.current = state.currentToolCall - } - else { - fullText += rawNewText // count as plaintext - console.log('adding raw a', rawNewText) - - } - } - else if (state.level === 'tool') { - if (tagName in (toolOfToolName[state.toolName]?.params ?? {})) { // valid param - state = { - level: 'param', - toolName: state.toolName, - paramName: tagName as ToolParamName, - currentToolCall: state.currentToolCall, - } - } - else { - // would normally be rawNewText, but we ignore all text inside tools - } - } - else if (state.level === 'param') { // cannot double nest - fullText += rawNewText // count as plaintext - console.log('adding raw b', rawNewText) - - } - - console.log('state1:', state.level, { toolName: (state as any).toolName, paramName: (state as any).paramName }) - - }; - - parser.onclosetag = (tagName) => { - const rawNewText = getRawNewText() - console.log('CLOSING', tagName) - console.log('state0:', state.level, { toolName: (state as any).toolName, paramName: (state as any).paramName }) - - - if (state.level === 'normal') { - fullText += rawNewText - console.log('adding raw A', rawNewText) - } - else if (state.level === 'tool') { - if (tagName === state.toolName) { // closed the tool - state.currentToolCall.isDone = true - state = { - level: 'normal', - } - } - else { // add as text - fullText += rawNewText - console.log('adding raw B', rawNewText) - } - } - else if (state.level === 'param') { - if (tagName === state.paramName) { // closed the param - state.currentToolCall.doneParams.push(state.paramName) - state = { - level: 'tool', - toolName: state.toolName, - currentToolCall: state.currentToolCall, - } - } - else { - fullText += rawNewText - console.log('adding raw C', rawNewText) - - } - } - console.log('state1:', state.level, { toolName: (state as any).toolName, paramName: (state as any).paramName }) - - - }; - - - parser.ontext = (text) => { - if (state.level === 'normal') { - fullText += text - } - // start param - else if (state.level === 'tool') { - // ignore all text in a tool, all text should go in the param tags inside it - } - else if (state.level === 'param') { - if (!(state.paramName in state.currentToolCall.rawParams)) state.currentToolCall.rawParams[state.paramName] = '' - state.currentToolCall.rawParams[state.paramName] += text - } - } - + let latestToolCall: RawToolCallObj | undefined = undefined + let foundOpenTag: { idx: number, toolName: ToolName } | null = null + let openToolTagBuffer = '' // the characters we've seen so far that come after a < with no space afterwards, not yet added to fullText let prevFullTextLen = 0 const newOnText: OnText = (params) => { @@ -282,13 +278,55 @@ export const extractToolsWrapper = ( prevFullTextLen = params.fullText.length trueFullText = params.fullText - parser.write(newText) + // console.log('NEWTEXT', JSON.stringify(newText)) + + + if (foundOpenTag === null) { + const newFullText = openToolTagBuffer + newText + // ensure the code below doesn't run if only half a tag has been written + const isPartial = findPartiallyWrittenToolTagAtEnd(newFullText, toolOpenTags) + if (isPartial) { + // console.log('--- partial!!!') + openToolTagBuffer += newText + } + // if no tooltag is partially written at the end, attempt to get the index + else { + // we will instantly retroactively remove this if it's a tag match + fullText += openToolTagBuffer + openToolTagBuffer = '' + fullText += newText + + const i = findIndexOfAny(fullText, toolOpenTags) + if (i !== null) { + const [idx, toolTag] = i + const toolName = toolTag.substring(1, toolTag.length - 1) as ToolName + // console.log('found ', toolName) + foundOpenTag = { idx, toolName } + + // do not count anything at or after i in fullText + fullText = fullText.substring(0, idx) + } + + + } + } + + // toolTagIdx is not null, so parse the XML + if (foundOpenTag !== null) { + latestToolCall = parseXMLPrefixToToolCall( + foundOpenTag.toolName, + trueFullText.substring(foundOpenTag.idx, Infinity), + toolOfToolName, + ) + + } + + - // firstToolCallRef.current === state.currentToolCall is always true onText({ ...params, fullText, - toolCall: firstToolCallRef.current, + toolCall: latestToolCall, }); }; @@ -298,16 +336,7 @@ export const extractToolsWrapper = ( newOnText({ ...params }) fullText = fullText.trimEnd() - const toolCall = firstToolCallRef.current - if (toolCall) { - // trim off all whitespace at and before first \n and after last \n for each param - for (const p in toolCall.rawParams) { - const paramName = p as ToolParamName - const orig = toolCall.rawParams[paramName] - if (orig === undefined) continue - toolCall.rawParams[paramName] = trimBeforeAndAfterNewLines(orig) - } - } + const toolCall = latestToolCall // console.log('final message!!!', trueFullText) // console.log('----- returning ----\n', fullText) @@ -321,7 +350,7 @@ export const extractToolsWrapper = ( -// trim all whitespace up until the first newline, and all whitespace after the last newline +// trim all whitespace up until the first newline, and all whitespace up until the last newline const trimBeforeAndAfterNewLines = (s: string) => { if (!s) return s; diff --git a/src/vs/workbench/contrib/void/electron-main/llmMessage/sax.ts b/src/vs/workbench/contrib/void/electron-main/llmMessage/sax.ts deleted file mode 100644 index 0d65e943..00000000 --- a/src/vs/workbench/contrib/void/electron-main/llmMessage/sax.ts +++ /dev/null @@ -1,150 +0,0 @@ -/*-------------------------------------------------------------------------------------- - * Copyright 2025 Glass Devtools, Inc. All rights reserved. - * Licensed under the Apache License, Version 2.0. See LICENSE.txt for more information. - *--------------------------------------------------------------------------------------*/ - -// Define options for the parser. -export interface SaxParserOptions { - lowercase?: boolean; -} - -// Define the structure for a parsed node. -export interface SaxNode { - name: string; - attributes: { [key: string]: string }; -} - -// Define the interface for the SAX-like parser. -export interface SaxParser { - // Event handlers that can be set by the consumer. - onopentag: ((node: SaxNode) => void) | null; - ontext: ((text: string) => void) | null; - onclosetag: ((tagName: string) => void) | null; - // Properties to track current positions (used for raw text extraction). - startTagPosition: number; - position: number; - // Processes a new chunk of text. - write(chunk: string): void; -} - -/** - * Creates a minimal, event-driven SAX-like parser. - * - * @param options An object of type `SaxParserOptions`. Passing `{ lowercase: true }` will force all tag names to be lower-cased. - * @returns A parser object implementing the `SaxParser` interface. - */ -export function createSaxParser(options: SaxParserOptions = {}): SaxParser { - // Buffer to hold any leftover text (part of an incomplete tag). - let buffer: string = ''; - // Global counter to track the total processed characters. - let globalPos: number = 0; - - const parser: SaxParser = { - onopentag: null, - ontext: null, - onclosetag: null, - startTagPosition: 0, - position: 0, - - write(chunk: string): void { - // Set the starting position before processing the new chunk. - this.startTagPosition = globalPos; - buffer += chunk; - globalPos += chunk.length; - // Set the current position to the end of the processed chunk. - this.position = globalPos - 1; - - let cursor = 0; - // Flag to indicate if an incomplete tag was found. - let incompleteTagFound = false; - // This will mark the position in the buffer where the incomplete tag starts. - let incompleteStart = 0; - - while (cursor < buffer.length) { - // Look for the next opening '<' character. - const ltIndex = buffer.indexOf('<', cursor); - if (ltIndex === -1) { - // No more tags found in the current buffer. - if (cursor < buffer.length && this.ontext) { - this.ontext(buffer.substring(cursor)); - } - // All content is processed. - buffer = ''; - cursor = buffer.length; - break; - } - - // Emit any text between the current cursor and the opening tag. - if (ltIndex > cursor && this.ontext) { - this.ontext(buffer.substring(cursor, ltIndex)); - } - - // Look for the closing '>' character starting from the found '<'. - const gtIndex = buffer.indexOf('>', ltIndex); - if (gtIndex === -1) { - // Incomplete tag detected. - incompleteTagFound = true; - // Save the starting point of the incomplete tag. - incompleteStart = ltIndex; - break; - } - - // Extract the tag content (excluding the '<' and '>'). - let tagContent = buffer.substring(ltIndex + 1, gtIndex).trim(); - if (!tagContent) { - cursor = gtIndex + 1; - continue; - } - - // Check if this is a closing tag (starts with '/'). - if (tagContent[0] === '/') { - let tagName = tagContent.substring(1).trim(); - if (options.lowercase && tagName) { - tagName = tagName.toLowerCase(); - } - if (this.onclosetag) { - this.onclosetag(tagName); - } - } else { - // Handle self-closing tags (ending with '/'). - let selfClosing = false; - if (tagContent[tagContent.length - 1] === '/') { - selfClosing = true; - tagContent = tagContent.slice(0, -1).trim(); - } - // Determine the tag name (first word before any whitespace). - const spaceIndex = tagContent.indexOf(' '); - let tagName = - spaceIndex !== -1 - ? tagContent.substring(0, spaceIndex).trim() - : tagContent; - if (options.lowercase && tagName) { - tagName = tagName.toLowerCase(); - } - // Emit an open tag event. - if (this.onopentag) { - const node: SaxNode = { name: tagName, attributes: {} }; - this.onopentag(node); - } - // If it’s a self-closing tag, immediately emit a close tag event. - if (selfClosing && this.onclosetag) { - this.onclosetag(tagName); - } - } - // Move the cursor past the current tag. - cursor = gtIndex + 1; - } - - // If an incomplete tag was detected, preserve it. - if (incompleteTagFound) { - // Keep the incomplete portion starting from the '<' - buffer = buffer.substring(incompleteStart); - } else { - // Otherwise, remove all processed content. - buffer = buffer.substring(cursor); - } - }, - }; - - return parser; -}