misc fixes + clarify displayContent

src/vs/workbench/contrib/void/browser/chatThreadService.ts

@@ -68,15 +68,17 @@ const toLLMChatMessages = (chatMessages: ChatMessage[]): LLMChatMessage[] => {
 
 	for (const c of chatMessages) {
 		if (c.role === 'assistant')
-			llmChatMessages.push({ role: c.role, content: c.content, anthropicReasoning: c.anthropicReasoning })
+			llmChatMessages.push({ role: c.role, content: c.displayContent, anthropicReasoning: c.anthropicReasoning })
 		// merge all tool/user messages into one big user message
 		else if (c.role === 'user' || c.role === 'tool') {
-			if (llmChatMessages.length === 0 || llmChatMessages[llmChatMessages.length - 1].role !== 'user') {
+			if (c.role === 'tool')
+				c.content = `TOOL_RESULT (${c.name}):\n${c.content}`
+
+			if (llmChatMessages.length === 0 || llmChatMessages[llmChatMessages.length - 1].role !== 'user')
 				llmChatMessages.push({ role: 'user', content: c.content })
-			}
-			else {
+			else
 				llmChatMessages[llmChatMessages.length - 1].content += '\n\n' + c.content
-			}
+
 		}
 		else if (c.role === 'interrupted_streaming_tool') { // pass
 		}
@@ -146,7 +148,7 @@ export type ThreadStreamState = {
 
 		// streaming related - when streaming message
 		streamingToken?: string;
-		messageSoFar?: string;
+		displayContentSoFar?: string;
 		reasoningSoFar?: string;
 		toolCallSoFar?: RawToolCallObj;
 	}
@@ -519,7 +521,7 @@ class ChatThreadService extends Disposable implements IChatThreadService {
 		const isRunning = this.streamState[threadId]?.isRunning
 		if (isRunning === 'LLM') {
 			// abort the stream first so it doesn't change any state
-			const messageSoFar = this.streamState[threadId]?.messageSoFar ?? ''
+			const displayContentSoFar = this.streamState[threadId]?.displayContentSoFar ?? ''
 			const reasoningSoFar = this.streamState[threadId]?.reasoningSoFar ?? ''
 			const toolCallSoFar = this.streamState[threadId]?.toolCallSoFar
 			console.log('toolInProgress', toolCallSoFar)
@@ -527,7 +529,7 @@ class ChatThreadService extends Disposable implements IChatThreadService {
 			const llmCancelToken = this.streamState[threadId]?.streamingToken
 			if (llmCancelToken !== undefined) { this._llmMessageService.abort(llmCancelToken) }
 
-			this._addMessageToThread(threadId, { role: 'assistant', content: messageSoFar, reasoning: reasoningSoFar, anthropicReasoning: null })
+			this._addMessageToThread(threadId, { role: 'assistant', displayContent: displayContentSoFar, reasoning: reasoningSoFar, anthropicReasoning: null })
 
 			if (toolCallSoFar) {
 				this._addMessageToThread(threadId, { role: 'interrupted_streaming_tool', name: toolCallSoFar.name })
@@ -716,19 +718,19 @@ class ChatThreadService extends Disposable implements IChatThreadService {
 				modelSelectionOptions,
 				logging: { loggingName: `Chat - ${chatMode}`, loggingExtras: { threadId, nMessagesSent, chatMode } },
 				onText: ({ fullText, fullReasoning, toolCall }) => {
-					this._setStreamState(threadId, { messageSoFar: fullText, reasoningSoFar: fullReasoning, toolCallSoFar: toolCall }, 'merge')
+					this._setStreamState(threadId, { displayContentSoFar: fullText, reasoningSoFar: fullReasoning, toolCallSoFar: toolCall }, 'merge')
 				},
 				onFinalMessage: async ({ fullText, fullReasoning, toolCall, anthropicReasoning, }) => {
-					this._addMessageToThread(threadId, { role: 'assistant', content: fullText, reasoning: fullReasoning, anthropicReasoning })
-					this._setStreamState(threadId, { messageSoFar: undefined, reasoningSoFar: undefined, streamingToken: undefined, toolCallSoFar: undefined }, 'merge')
+					this._addMessageToThread(threadId, { role: 'assistant', displayContent: fullText, reasoning: fullReasoning, anthropicReasoning })
+					this._setStreamState(threadId, { displayContentSoFar: undefined, reasoningSoFar: undefined, streamingToken: undefined, toolCallSoFar: undefined }, 'merge')
 					console.log('tool call!!', toolCall)
 					resMessageIsDonePromise(toolCall) // resolve with tool calls
 				},
 				onError: (error) => {
-					const messageSoFar = this.streamState[threadId]?.messageSoFar ?? ''
+					const messageSoFar = this.streamState[threadId]?.displayContentSoFar ?? ''
 					const reasoningSoFar = this.streamState[threadId]?.reasoningSoFar ?? ''
 					// add assistant's message to chat history, and clear selection
-					this._addMessageToThread(threadId, { role: 'assistant', content: messageSoFar, reasoning: reasoningSoFar, anthropicReasoning: null })
+					this._addMessageToThread(threadId, { role: 'assistant', displayContent: messageSoFar, reasoning: reasoningSoFar, anthropicReasoning: null })
 					this._setStreamState(threadId, { error }, 'set')
 					resMessageIsDonePromise()
 				},

src/vs/workbench/contrib/void/browser/react/src/sidebar-tsx/SidebarChat.tsx

@@ -1075,7 +1075,7 @@ const AssistantMessageComponent = ({ chatMessage, isCheckpointGhost, isCommitted
 
 	const reasoningStr = chatMessage.reasoning?.trim() || null
 	const hasReasoning = !!reasoningStr
-	const isDoneReasoning = !!chatMessage.content
+	const isDoneReasoning = !!chatMessage.displayContent
 	const thread = chatThreadsService.getCurrentThread()
 
 
@@ -1084,7 +1084,7 @@ const AssistantMessageComponent = ({ chatMessage, isCheckpointGhost, isCommitted
 		messageIdx: messageIdx,
 	}
 
-	const isEmpty = !chatMessage.content && !chatMessage.reasoning
+	const isEmpty = !chatMessage.displayContent && !chatMessage.reasoning
 	if (isEmpty) return null
 
 	return <>
@@ -1108,7 +1108,7 @@ const AssistantMessageComponent = ({ chatMessage, isCheckpointGhost, isCommitted
 		<div className={`${isCheckpointGhost ? 'opacity-50' : ''}`}>
 			<ProseWrapper>
 				<ChatMarkdownRender
-					string={chatMessage.content || ''}
+					string={chatMessage.displayContent || ''}
 					chatMessageLocation={chatMessageLocation}
 					isApplyEnabled={true}
 					isLinkDetectionEnabled={true}
@@ -2005,7 +2005,7 @@ export const SidebarChat = () => {
 	const currThreadStreamState = useChatThreadsStreamState(chatThreadsState.currentThreadId)
 	const isRunning = currThreadStreamState?.isRunning
 	const latestError = currThreadStreamState?.error
-	const messageSoFar = currThreadStreamState?.messageSoFar
+	const displayContentSoFar = currThreadStreamState?.displayContentSoFar
 	const reasoningSoFar = currThreadStreamState?.reasoningSoFar
 
 	const toolCallSoFar = currThreadStreamState?.toolCallSoFar
@@ -2082,13 +2082,13 @@ export const SidebarChat = () => {
 	}, [previousMessages, isRunning, threadId])
 
 	const streamingChatIdx = previousMessagesHTML.length
-	const currStreamingMessageHTML = reasoningSoFar || messageSoFar || isRunning ?
+	const currStreamingMessageHTML = reasoningSoFar || displayContentSoFar || isRunning ?
 		<ChatBubble
 			key={getChatBubbleId(threadId, streamingChatIdx)}
 			currCheckpointIdx={currCheckpointIdx} // if streaming, can't be the case
 			chatMessage={{
 				role: 'assistant',
-				content: messageSoFar ?? '',
+				displayContent: displayContentSoFar ?? '',
 				reasoning: reasoningSoFar ?? '',
 				anthropicReasoning: null,
 			}}
@@ -2112,7 +2112,7 @@ export const SidebarChat = () => {
 			w-full h-full
 			overflow-x-hidden
 			overflow-y-auto
-			${previousMessagesHTML.length === 0 && !messageSoFar ? 'hidden' : ''}
+			${previousMessagesHTML.length === 0 && !displayContentSoFar ? 'hidden' : ''}
 		`}
 	>
 		{/* previous messages */}

src/vs/workbench/contrib/void/common/chatThreadServiceTypes.ts

@@ -56,7 +56,7 @@ export type ChatMessage =
 		}
 	} | {
 		role: 'assistant';
-		content: string; // content received from LLM  - allowed to be '', will be replaced with (empty)
+		displayContent: string; // content received from LLM  - allowed to be '', will be replaced with (empty)
 		reasoning: string; // reasoning from the LLM, used for step-by-step thinking
 
 		anthropicReasoning: AnthropicReasoning[] | null; // anthropic reasoning

src/vs/workbench/contrib/void/electron-main/llmMessage/extractGrammar.ts

@@ -29,6 +29,7 @@ export const extractReasoningWrapper = (
 	}
 
 	const newOnText: OnText = ({ fullText: fullText_, ...p }) => {
+
 		// until found the first think tag, keep adding to fullText
 		if (!foundTag1) {
 			const endsWithTag1 = endsWithAnyPrefixOf(fullText_, thinkTags[0])
@@ -293,6 +294,9 @@ export const extractToolsWrapper = (
 
 	const newOnFinalMessage: OnFinalMessage = (params) => {
 		// treat like just got text before calling onFinalMessage (or else we sometimes miss the final chunk that's new to finalMessage)
+		console.log('final message!!!', trueFullText)
+		console.log('----- returning ----\n', fullText)
+		console.log('----- tools ----\n', JSON.stringify(currentToolCalls, null, 2))
 		newOnText({ ...params })
 
 		console.log('final message!!!', trueFullText)
@@ -300,7 +304,7 @@ export const extractToolsWrapper = (
 		console.log('----- tools ----\n', JSON.stringify(currentToolCalls, null, 2))
 
 		fullText = fullText.trimEnd()
-		const toolCall = currentToolCalls[0]
+		const toolCall = currentToolCalls.length > 0 ? currentToolCalls[0] : undefined
 		if (toolCall) {
 			// trim off all whitespace at and before first \n and after last \n for each param
 			for (const paramName in toolCall.rawParams) {
@@ -309,7 +313,9 @@ export const extractToolsWrapper = (
 				toolCall.rawParams[paramName] = trimBeforeAndAfterNewLines(orig)
 			}
 		}
-		onFinalMessage({ ...params, fullText, toolCall: currentToolCalls.length > 0 ? currentToolCalls[0] : undefined })
+		console.log('----- toolCall ----\n', JSON.stringify(toolCall, null, 2))
+
+		onFinalMessage({ ...params, fullText, toolCall: toolCall })
 	}
 	return { newOnText, newOnFinalMessage };
 }

src/vs/workbench/contrib/void/electron-main/llmMessage/sax.ts

@@ -54,30 +54,38 @@ export function createSaxParser(options: SaxParserOptions = {}): SaxParser {
 			// Set the current position to the end of the processed chunk.
 			this.position = globalPos - 1;
 
-			let cursor: number = 0;
+			let cursor = 0;
+			// Flag to indicate if an incomplete tag was found.
+			let incompleteTagFound = false;
+			// This will mark the position in the buffer where the incomplete tag starts.
+			let incompleteStart = 0;
+
 			while (cursor < buffer.length) {
 				// Look for the next opening '<' character.
 				const ltIndex = buffer.indexOf('<', cursor);
 				if (ltIndex === -1) {
-					// No more tags found. Emit any remaining text as a text node.
+					// No more tags found in the current buffer.
 					if (cursor < buffer.length && this.ontext) {
 						this.ontext(buffer.substring(cursor));
 					}
-					// Clear the buffer since all content is processed.
+					// All content is processed.
 					buffer = '';
+					cursor = buffer.length;
 					break;
 				}
 
-				// Emit any text that appears before the tag.
+				// Emit any text between the current cursor and the opening tag.
 				if (ltIndex > cursor && this.ontext) {
 					this.ontext(buffer.substring(cursor, ltIndex));
 				}
 
-				// Look for the closing '>' character.
+				// Look for the closing '>' character starting from the found '<'.
 				const gtIndex = buffer.indexOf('>', ltIndex);
 				if (gtIndex === -1) {
-					// Incomplete tag detected—retain the remaining content in the buffer.
-					buffer = buffer.substring(ltIndex);
+					// Incomplete tag detected.
+					incompleteTagFound = true;
+					// Save the starting point of the incomplete tag.
+					incompleteStart = ltIndex;
 					break;
 				}
 
@@ -98,24 +106,27 @@ export function createSaxParser(options: SaxParserOptions = {}): SaxParser {
 						this.onclosetag(tagName);
 					}
 				} else {
-					// Check for self-closing tags (ending with '/').
+					// Handle self-closing tags (ending with '/').
 					let selfClosing = false;
 					if (tagContent[tagContent.length - 1] === '/') {
 						selfClosing = true;
 						tagContent = tagContent.slice(0, -1).trim();
 					}
-					// Determine the tag name (first word before whitespace).
+					// Determine the tag name (first word before any whitespace).
 					const spaceIndex = tagContent.indexOf(' ');
-					let tagName = (spaceIndex !== -1 ? tagContent.substring(0, spaceIndex) : tagContent).trim();
+					let tagName =
+						spaceIndex !== -1
+							? tagContent.substring(0, spaceIndex).trim()
+							: tagContent;
 					if (options.lowercase && tagName) {
 						tagName = tagName.toLowerCase();
 					}
-					// Call onopentag with a minimal node object.
+					// Emit an open tag event.
 					if (this.onopentag) {
 						const node: SaxNode = { name: tagName, attributes: {} };
 						this.onopentag(node);
 					}
-					// If the tag is self-closing, immediately emit the closing tag event.
+					// If it’s a self-closing tag, immediately emit a close tag event.
 					if (selfClosing && this.onclosetag) {
 						this.onclosetag(tagName);
 					}
@@ -124,10 +135,15 @@ export function createSaxParser(options: SaxParserOptions = {}): SaxParser {
 				cursor = gtIndex + 1;
 			}
 
-			// Remove any content already processed from the buffer.
-			buffer = buffer.slice(cursor);
-		}
-
+			// If an incomplete tag was detected, preserve it.
+			if (incompleteTagFound) {
+				// Keep the incomplete portion starting from the '<'
+				buffer = buffer.substring(incompleteStart);
+			} else {
+				// Otherwise, remove all processed content.
+				buffer = buffer.substring(cursor);
+			}
+		},
 	};
 
 	return parser;