Initial draft for speculative edits

extensions/void/src/common/ctrlL.ts

@@ -0,0 +1,676 @@
+import * as vscode from 'vscode';
+import { OnFinalMessage, OnText, sendLLMMessage, SetAbort } from "./sendLLMMessage"
+import { VoidConfig } from '../sidebar/contextForConfig';
+
+const generateDiffInstructions = `
+You are a coding assistant. You are given a list of relevant files \`files\`, a selection that the user is making \`selection\`, and instructions to follow \`instructions\`.
+
+Please edit the selected file following the user's instructions (or, if appropriate, answer their question instead).
+
+All changes made to files must be outputted in unified diff format.
+Unified diff format instructions:
+1. Each diff must begin with \`\`\`@@ ... @@\`\`\`.
+2. Each line must start with a \`+\` or \`-\` or \` \` symbol.
+3. Make diffs more than a few lines.
+4. Make high-level diffs rather than many one-line diffs.
+
+Here's an example of unified diff format:
+
+\`\`\`
+@@ ... @@
+-def factorial(n):
+-    if n == 0:
+-        return 1
+-    else:
+-        return n * factorial(n-1)
++def factorial(number):
++    if number == 0:
++        return 1
++    else:
++        return number * factorial(number-1)
+\`\`\`
+
+Please create high-level diffs where you group edits together if they are near each other, like in the above example. Another way to represent the above example is to make many small line edits. However, this is less preferred, because the edits are not high-level. The edits are close together and should be grouped:
+
+\`\`\`
+@@ ... @@ # This is less preferred because edits are close together and should be grouped:
+-def factorial(n):
++def factorial(number):
+-    if n == 0:
++    if number == 0:
+         return 1
+     else:
+-        return n * factorial(n-1)
++        return number * factorial(number-1)
+\`\`\`
+
+# Example 1:
+
+FILES
+selected file \`test.ts\`:
+\`\`\`
+x = 1
+
+{{selection}}
+
+z = 3
+\`\`\`
+
+SELECTION
+\`\`\`const y = 2\`\`\`
+
+INSTRUCTIONS
+\`\`\`y = 3\`\`\`
+
+EXPECTED RESULT
+Following the instructions, we should change the selection from \`\`\`y = 2\`\`\` to \`\`\`y = 3\`\`\`. Here is the expected output diff:
+\`\`\`
+@@ ... @@
+-x = 1
+-
+-y = 2
++x = 1
++
++y = 3
+\`\`\`
+
+# Example 2:
+
+FILES
+selected file \`Sidebar.tsx\`:
+\`\`\`
+import React from 'react';
+import styles from './Sidebar.module.css';
+
+interface SidebarProps {
+  items: { label: string; href: string }[];
+  onItemSelect?: (label: string) => void;
+  onExtraButtonClick?: () => void;
+}
+
+const Sidebar: React.FC<SidebarProps> = ({ items, onItemSelect, onExtraButtonClick }) => {
+  return (
+    <div className={styles.sidebar}>
+      <ul>
+        {items.map((item, index) => (
+          <li key={index}>
+             {{selection}}
+              className={styles.sidebarButton}
+              onClick={() => onItemSelect?.(item.label)}
+            >
+              {item.label}
+            </button>
+          </li>
+        ))}
+      </ul>
+      <button className={styles.extraButton} onClick={onExtraButtonClick}>
+        Extra Action
+      </button>
+    </div>
+  );
+};
+
+export default Sidebar;
+\`\`\`
+
+SELECTION
+\`\`\`             <button\`\`\`
+
+INSTRUCTIONS
+\`\`\`make all the buttons like this into divs\`\`\`
+
+EXPECTED OUTPUT
+
+Following the instructions, we should change all the buttons like the one selected into a div component. Here is the result:
+\`\`\`
+@@ ... @@
+-<div className={styles.sidebar}>
+-<ul>
+-  {items.map((item, index) => (
+-	<li key={index}>
+-	  <button
+-		className={styles.sidebarButton}
+-		onClick={() => onItemSelect?.(item.label)}
+-	  >
+-		{item.label}
+-	  </button>
+-	</li>
+-  ))}
+-</ul>
+-<button className={styles.extraButton} onClick={onExtraButtonClick}>
+-  Extra Action
+-</button>
+-</div>
++<div className={styles.sidebar}>
++<ul>
++  {items.map((item, index) => (
++	<li key={index}>
++	  <div
++		className={styles.sidebarButton}
++		onClick={() => onItemSelect?.(item.label)}
++	  >
++		{item.label}
++	  </div>
++	</li>
++  ))}
++</ul>
++<div className={styles.extraButton} onClick={onExtraButtonClick}>
++  Extra Action
++</div>
++</div>
+\`\`\`
+`;
+
+
+const searchDiffChunkInstructions = `
+You are a coding assistant that applies a diff to a file. You are given a diff \`diff\`, a list of files \`files\` to apply the diff to, and a selection \`selection\` that you are currently considering in the file.
+
+Determine whether you should modify ANY PART of the selection \`selection\` following the \`diff\`. Return \`true\` if you should modify any part of the selection, and \`false\` if you should not modify any part of it.
+
+# Example 1:
+
+FILES
+selected file \`Sidebar.tsx\`:
+\`\`\`
+import React from 'react';
+import styles from './Sidebar.module.css';
+
+interface SidebarProps {
+  items: { label: string; href: string }[];
+  onItemSelect?: (label: string) => void;
+  onExtraButtonClick?: () => void;
+}
+
+const Sidebar: React.FC<SidebarProps> = ({ items, onItemSelect, onExtraButtonClick }) => {
+  return (
+    <div className={styles.sidebar}>
+      <ul>
+        {items.map((item, index) => (
+          <li key={index}>
+            <button
+              className={styles.sidebarButton}
+              onClick={() => onItemSelect?.(item.label)}
+            >
+              {item.label}
+            </button>
+          </li>
+        ))}
+      </ul>
+      <button className={styles.extraButton} onClick={onExtraButtonClick}>
+        Extra Action
+      </button>
+    </div>
+  );
+};
+
+export default Sidebar;
+\`\`\`
+
+DIFF
+\`\`\`
+@@ ... @@
+-<div className={styles.sidebar}>
+-<ul>
+-  {items.map((item, index) => (
+-	<li key={index}>
+-	  <button
+-		className={styles.sidebarButton}
+-		onClick={() => onItemSelect?.(item.label)}
+-	  >
+-		{item.label}
+-	  </button>
+-	</li>
+-  ))}
+-</ul>
+-<button className={styles.extraButton} onClick={onExtraButtonClick}>
+-  Extra Action
+-</button>
+-</div>
++<div className={styles.sidebar}>
++<ul>
++  {items.map((item, index) => (
++	<li key={index}>
++	  <div
++		className={styles.sidebarButton}
++		onClick={() => onItemSelect?.(item.label)}
++	  >
++		{item.label}
++	  </div>
++	</li>
++  ))}
++</ul>
++<div className={styles.extraButton} onClick={onExtraButtonClick}>
++  Extra Action
++</div>
++</div>
+\`\`\`
+
+SELECTION
+\`\`\`
+import React from 'react';
+import styles from './Sidebar.module.css';
+
+interface SidebarProps {
+  items: { label: string; href: string }[];
+  onItemSelect?: (label: string) => void;
+  onExtraButtonClick?: () => void;
+}
+
+const Sidebar: React.FC<SidebarProps> = ({ items, onItemSelect, onExtraButtonClick }) => {
+  return (
+    <div className={styles.sidebar}>
+      <ul>
+        {items.map((item, index) => (
+\`\`\`
+
+EXPECTED RESULT
+The expected output is \`true\`, because the diff begins on the line with \`<div className={styles.sidebar}>\` and this line is present in the selection.
+
+\`true\`
+`
+
+
+const searchDiffLineInstructions = `
+You are a coding assistant that applies a diff to a file. You are given a diff \`diff\`, a list of files \`files\` to apply the diff to, and a selection \`selection\` that you are currently considering in the file.
+
+Determine whether you should modify ANY PART of the selection \`selection\` following the \`diff\`. Return \`true\` if you should modify any part of the selection, and \`false\` if you should not modify any part of it.
+
+# Example 1:
+
+FILES
+selected file \`Sidebar.tsx\`:
+\`\`\`
+import React from 'react';
+import styles from './Sidebar.module.css';
+
+interface SidebarProps {
+  items: { label: string; href: string }[];
+  onItemSelect?: (label: string) => void;
+  onExtraButtonClick?: () => void;
+}
+
+const Sidebar: React.FC<SidebarProps> = ({ items, onItemSelect, onExtraButtonClick }) => {
+  return (
+    <div className={styles.sidebar}>
+      <ul>
+        {items.map((item, index) => (
+          <li key={index}>
+            <button
+              className={styles.sidebarButton}
+              onClick={() => onItemSelect?.(item.label)}
+            >
+              {item.label}
+            </button>
+          </li>
+        ))}
+      </ul>
+      <button className={styles.extraButton} onClick={onExtraButtonClick}>
+        Extra Action
+      </button>
+    </div>
+  );
+};
+
+export default Sidebar;
+\`\`\`
+
+DIFF
+\`\`\`
+@@ ... @@
+-<div className={styles.sidebar}>
+-<ul>
+-  {items.map((item, index) => (
+-	<li key={index}>
+-	  <button
+-		className={styles.sidebarButton}
+-		onClick={() => onItemSelect?.(item.label)}
+-	  >
+-		{item.label}
+-	  </button>
+-	</li>
+-  ))}
+-</ul>
+-<button className={styles.extraButton} onClick={onExtraButtonClick}>
+-  Extra Action
+-</button>
+-</div>
++<div className={styles.sidebar}>
++<ul>
++  {items.map((item, index) => (
++	<li key={index}>
++	  <div
++		className={styles.sidebarButton}
++		onClick={() => onItemSelect?.(item.label)}
++	  >
++		{item.label}
++	  </div>
++	</li>
++  ))}
++</ul>
++<div className={styles.extraButton} onClick={onExtraButtonClick}>
++  Extra Action
++</div>
++</div>
+\`\`\`
+
+SELECTION
+\`\`\`
+import React from 'react';
+import styles from './Sidebar.module.css';
+
+interface SidebarProps {
+  items: { label: string; href: string }[];
+  onItemSelect?: (label: string) => void;
+  onExtraButtonClick?: () => void;
+}
+
+const Sidebar: React.FC<SidebarProps> = ({ items, onItemSelect, onExtraButtonClick }) => {
+  return (
+    <div className={styles.sidebar}>
+      <ul>
+        {items.map((item, index) => (
+\`\`\`
+
+EXPECTED RESULT
+The expected output is \`true\`, because the diff begins on the line with \`<div className={styles.sidebar}>\` and this line is present in the selection.
+
+\`true\`
+`
+
+
+
+const rewriteFileWithDiffInstructions = `
+You are a coding assistant that applies a diff to a file. You are given the original file \`original_file\`, a diff \`diff\`, and a new file that you are applying the diff to \`new_file\`.
+
+Please finish writing the new file \`new_file\`, according to the diff \`diff\`.
+
+Directions:
+1. Continue exactly where the new file \`new_file\` left off.
+2. Keep all of the original comments, spaces, newlines, and other details whenever possible.
+3. Note that in the diff \`diff\`, \`+\` lines represent additions, \`-\` lines represent removals, and space lines \` \` represent no change.
+
+# Example 1:
+
+ORIGINAL_FILE
+\`Sidebar.tsx\`:
+\`\`\`
+import React from 'react';
+import styles from './Sidebar.module.css';
+
+interface SidebarProps {
+  items: { label: string; href: string }[];
+  onItemSelect?: (label: string) => void;
+  onExtraButtonClick?: () => void;
+}
+
+const Sidebar: React.FC<SidebarProps> = ({ items, onItemSelect, onExtraButtonClick }) => {
+  return (
+    <div className={styles.sidebar}>
+      <ul>
+        {items.map((item, index) => (
+          <li key={index}>
+            <button
+              className={styles.sidebarButton}
+              onClick={() => onItemSelect?.(item.label)}
+            >
+              {item.label}
+            </button>
+          </li>
+        ))}
+      </ul>
+      <button className={styles.extraButton} onClick={onExtraButtonClick}>
+        Extra Action
+      </button>
+    </div>
+  );
+};
+
+export default Sidebar;
+\`\`\`
+
+DIFF
+\`\`\`
+@@ ... @@
+-<div className={styles.sidebar}>
+-<ul>
+-  {items.map((item, index) => (
+-	<li key={index}>
+-	  <button
+-		className={styles.sidebarButton}
+-		onClick={() => onItemSelect?.(item.label)}
+-	  >
+-		{item.label}
+-	  </button>
+-	</li>
+-  ))}
+-</ul>
+-<button className={styles.extraButton} onClick={onExtraButtonClick}>
+-  Extra Action
+-</button>
+-</div>
++<div className={styles.sidebar}>
++<ul>
++  {items.map((item, index) => (
++	<li key={index}>
++	  <div
++		className={styles.sidebarButton}
++		onClick={() => onItemSelect?.(item.label)}
++	  >
++		{item.label}
++	  </div>
++	</li>
++  ))}
++</ul>
++<div className={styles.extraButton} onClick={onExtraButtonClick}>
++  Extra Action
++</div>
++</div>
+\`\`\`
+
+NEW_FILE
+\`\`\`
+import React from 'react';
+import styles from './Sidebar.module.css';
+
+interface SidebarProps {
+  items: { label: string; href: string }[];
+  onItemSelect?: (label: string) => void;
+  onExtraButtonClick?: () => void;
+}
+
+const Sidebar: React.FC<SidebarProps> = ({ items, onItemSelect, onExtraButtonClick }) => {
+  return (
+\`\`\`
+
+EXPECTED RESULT
+The expected output should complete the new file \`new_file\`, following the diff \`diff\`. Here is the expected output:
+\`\`\`
+    <div className={styles.sidebar}>
+      <ul>
+        {items.map((item, index) => (
+          <li key={index}>
+            <div
+              className={styles.sidebarButton}
+              onClick={() => onItemSelect?.(item.label)}
+            >
+              {item.label}
+            </div>
+          </li>
+        ))}
+      </ul>
+      <div className={styles.extraButton} onClick={onExtraButtonClick}>
+        Extra Action
+      </div>
+    </div>
+  );
+};
+
+export default Sidebar;
+\`\`\`
+`
+
+
+type Res<T> = ((value: T) => void)
+
+
+const rewriteFileWithDiff = ({ fileUri, originalFileStr, newFileStr, diff, voidConfig, onText, setAbort }: { fileUri: vscode.Uri, originalFileStr: string, newFileStr: string, diff: string, voidConfig: VoidConfig, onText: OnText, setAbort: SetAbort }) => {
+
+	const EXTRA_TOKENS = 20
+
+	const promptContent = `ORIGINAL_FILE
+\`\`\`
+${originalFileStr}
+\`\`\`
+
+DIFF
+\`\`\`
+${diff}
+\`\`\`
+
+INSTRUCTIONS
+Please finish writing the new file \`NEW_FILE\`. When
+
+NEW_FILE
+\`\`\`
+${newFileStr}
+\`\`\`
+`
+	// create a promise that can be awaited
+	let res: Res<string> = () => { }
+	const promise = new Promise<string>((resolve, reject) => { res = resolve })
+
+
+	// make LLM rewrite file to include the diff
+	sendLLMMessage({
+		messages: [{ role: 'assistant', content: rewriteFileWithDiffInstructions, }, { role: 'assistant', content: promptContent, }],
+		onText,
+		onFinalMessage: (finalMessage) => { res(finalMessage) },
+		onError: () => { res(''); console.error('Error rewriting file with diff') },
+		voidConfig: {
+			...voidConfig,
+			default: {
+				// set `max_tokens` = (number of expected tokens) + (number of extra tokens)
+				maxTokens: Math.round((diff.split('\n').filter(l => !l.startsWith('-')).length) + EXTRA_TOKENS) + ''
+			}
+		},
+		setAbort,
+	})
+
+
+	return promise
+
+}
+
+const shouldApplyDiffToLine = async ({ diff, fileStr, lineStr, voidConfig, setAbort }: { diff: string, fileStr: string, lineStr: string, voidConfig: VoidConfig, setAbort: SetAbort }) => {
+
+	const promptContent = `DIFF
+\`\`\`
+${diff}
+\`\`\`
+
+FILES
+\`\`\`
+${fileStr}
+\`\`\`
+
+SELECTION
+\`\`\`${lineStr}\`\`\`
+
+Return \`true\` if this line should be modified, and \`false\` if it should not be modified.
+`
+
+	// create new promise
+	let res: Res<boolean> = () => { }
+	const promise = new Promise<boolean>((resolve, reject) => { res = resolve })
+
+	sendLLMMessage({
+		messages: [{ role: 'assistant', content: searchDiffLineInstructions, }, { role: 'assistant', content: promptContent, }],
+		onText: () => { },
+		onFinalMessage: (finalMessage) => {
+			const containsTrue = finalMessage
+				.slice(-10)
+				.toLowerCase()
+				.includes('true')
+			res(containsTrue)
+		},
+		onError: () => {
+			res(false);
+			console.error('Error applying diff to line')
+		},
+		voidConfig,
+		setAbort
+	})
+
+	return promise
+
+}
+
+
+
+// lazily applies the diff to the file
+// we chunk the text in the file, and ask an LLM whether it should edit each chunk
+const applyDiffLazily = async ({ fileUri, fileStr, diff, voidConfig, setAbort }: { fileUri: vscode.Uri, fileStr: string, diff: string, voidConfig: VoidConfig, setAbort: SetAbort }) => {
+
+	const CHUNK_SIZE = 20 // number of lines to search at a time
+
+	// read file content
+	const fileLines = fileStr.split('\n')
+	const completedLines = []
+
+	// search the file chunk-by-chunk
+	for (let chunkIdx = 0; chunkIdx * CHUNK_SIZE < fileLines.length; chunkIdx++) {
+
+		// get the chunk
+		const chunkStart = chunkIdx * CHUNK_SIZE
+		const chunkEnd = (chunkIdx + 1) * CHUNK_SIZE
+		const chunkLines = fileLines.slice(chunkStart, chunkEnd)
+		const chunkStr = chunkLines.join('\n');
+
+		// ask LLM if we should apply the diff to the chunk
+		let shouldApplyDiff = await shouldApplyDiffToChunk({ chunkStr, diff, fileUri, setAbort })
+		if (!shouldApplyDiff) { // should not change the chunk
+			completedLines.push(chunkStr);
+			// TODO update highlighting here
+			continue;
+		}
+
+		// search the chunk line-by-line
+		for (const lineStr of chunkLines) {
+
+			// ask LLM if we should apply the diff to the line
+			let shouldApplyDiff = await shouldApplyDiffToLine({ diff, fileStr, lineStr, voidConfig, setAbort })
+			if (!shouldApplyDiff) { // should not change the line
+				completedLines.push(lineStr);
+				// TODO update highlighting here
+				continue;
+			}
+
+			// ask LLM to apply the diff
+			const changeStr = await rewriteFileWithDiff({ // rewrite file with diff (if there is significant matchup with the original file, we stop rewriting)
+				originalFileStr: fileStr,
+				newFileStr: completedLines.join('\n'),
+				diff,
+				fileUri,
+				voidConfig,
+				onText: async (text) => {
+					// TODO! update highlighting here
+					// also make edits here
+
+				},
+				setAbort,
+			})
+			completedLines.push(changeStr)
+
+
+			// if there's matchup with the file, we stop rewriting
+			// TODO! otherwise keep rewriting until there is matchup
+
+		}
+
+	}
+
+}
+
+
+
+export { applyDiffLazily }

extensions/void/src/common/sendLLMMessage.ts

@@ -6,7 +6,11 @@ import { VoidConfig } from '../sidebar/contextForConfig';
 
 
 
-type OnText = (newText: string, fullText: string) => void
+export type OnText = (newText: string, fullText: string) => void
+
+export type OnFinalMessage = (input: string) => void
+
+export type SetAbort = (abort: () => void) => void
 
 export type LLMMessage = {
 	role: 'user' | 'assistant',
@@ -16,13 +20,11 @@ export type LLMMessage = {
 type SendLLMMessageFnTypeInternal = (params: {
 	messages: LLMMessage[],
 	onText: OnText,
-	onFinalMessage: (input: string) => void,
+	onFinalMessage: OnFinalMessage,
 	onError: (error: string) => void,
 	voidConfig: VoidConfig,
-})
-	=> {
-		abort: () => void
-	}
+	setAbort: SetAbort,
+}) => void
 
 type SendLLMMessageFnTypeExternal = (params: {
 	messages: LLMMessage[],
@@ -30,22 +32,22 @@ type SendLLMMessageFnTypeExternal = (params: {
 	onFinalMessage: (input: string) => void,
 	onError: (error: string) => void,
 	voidConfig: VoidConfig | null,
+	setAbort: SetAbort,
+
 })
-	=> {
-		abort: () => void
-	}
+	=> void
 
 
 
 
 // Anthropic
-const sendAnthropicMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinalMessage, onError, voidConfig }) => {
+const sendAnthropicMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinalMessage, onError, voidConfig, setAbort }) => {
 
 	const anthropic = new Anthropic({ apiKey: voidConfig.anthropic.apikey, dangerouslyAllowBrowser: true }); // defaults to process.env["ANTHROPIC_API_KEY"]
 
 	const stream = anthropic.messages.stream({
 		model: voidConfig.anthropic.model,
-		max_tokens: parseInt(voidConfig.anthropic.maxTokens),
+		max_tokens: parseInt(voidConfig.default.maxTokens),
 		messages: messages,
 	});
 
@@ -80,8 +82,7 @@ const sendAnthropicMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFi
 		// stream.controller.abort() // TODO need to test this to make sure it works, it might throw an error
 		did_abort = true
 	}
-
-	return { abort }
+	setAbort(abort)
 
 };
 
@@ -89,7 +90,7 @@ const sendAnthropicMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFi
 
 
 // OpenAI, OpenRouter, OpenAICompatible
-const sendOpenAIMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinalMessage, onError, voidConfig }) => {
+const sendOpenAIMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinalMessage, onError, voidConfig, setAbort }) => {
 
 	let didAbort = false
 	let fullText = ''
@@ -104,7 +105,7 @@ const sendOpenAIMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinal
 
 	if (voidConfig.default.whichApi === 'openAI') {
 		openai = new OpenAI({ apiKey: voidConfig.openAI.apikey, dangerouslyAllowBrowser: true });
-		options = { model: voidConfig.openAI.model, messages: messages, stream: true, }
+		options = { model: voidConfig.openAI.model, messages: messages, stream: true, max_completion_tokens: parseInt(voidConfig.default.maxTokens) }
 	}
 	else if (voidConfig.default.whichApi === 'openRouter') {
 		openai = new OpenAI({
@@ -114,11 +115,11 @@ const sendOpenAIMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinal
 				"X-Title": 'Void Editor', // Optional. Shows in rankings on openrouter.ai.
 			},
 		});
-		options = { model: voidConfig.openRouter.model, messages: messages, stream: true, }
+		options = { model: voidConfig.openRouter.model, messages: messages, stream: true, max_completion_tokens: parseInt(voidConfig.default.maxTokens) }
 	}
 	else if (voidConfig.default.whichApi === 'openAICompatible') {
 		openai = new OpenAI({ baseURL: voidConfig.openAICompatible.endpoint, apiKey: voidConfig.openAICompatible.apikey, dangerouslyAllowBrowser: true })
-		options = { model: voidConfig.openAICompatible.model, messages: messages, stream: true, }
+		options = { model: voidConfig.openAICompatible.model, messages: messages, stream: true, max_completion_tokens: parseInt(voidConfig.default.maxTokens) }
 	}
 	else {
 		console.error(`sendOpenAIMsg: invalid whichApi: ${voidConfig.default.whichApi}`)
@@ -156,12 +157,12 @@ const sendOpenAIMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinal
 			}
 		})
 
-	return { abort };
+	setAbort(abort)
 };
 
 
 // Ollama
-export const sendOllamaMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinalMessage, onError, voidConfig }) => {
+export const sendOllamaMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinalMessage, onError, voidConfig, setAbort }) => {
 
 	let didAbort = false
 	let fullText = ""
@@ -177,6 +178,7 @@ export const sendOllamaMsg: SendLLMMessageFnTypeInternal = ({ messages, onText,
 		model: voidConfig.ollama.model,
 		messages: messages,
 		stream: true,
+		options: { num_predict: parseInt(voidConfig.default.maxTokens) } // this is max_tokens
 	})
 		.then(async stream => {
 			abort = () => {
@@ -198,7 +200,7 @@ export const sendOllamaMsg: SendLLMMessageFnTypeInternal = ({ messages, onText,
 			onError(error)
 		})
 
-	return { abort };
+	setAbort(abort);
 };
 
 
@@ -207,7 +209,7 @@ export const sendOllamaMsg: SendLLMMessageFnTypeInternal = ({ messages, onText,
 // https://docs.greptile.com/api-reference/query
 // https://docs.greptile.com/quickstart#sample-response-streamed
 
-const sendGreptileMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinalMessage, onError, voidConfig }) => {
+const sendGreptileMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFinalMessage, onError, voidConfig, setAbort }) => {
 
 	let didAbort = false
 	let fullText = ''
@@ -226,7 +228,7 @@ const sendGreptileMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFin
 		body: JSON.stringify({
 			messages,
 			stream: true,
-			repositories: [voidConfig.greptile.repoinfo]
+			repositories: [voidConfig.greptile.repoinfo],
 		}),
 	})
 		// this is {message}\n{message}\n{message}...\n
@@ -268,28 +270,26 @@ const sendGreptileMsg: SendLLMMessageFnTypeInternal = ({ messages, onText, onFin
 			onError(e)
 		});
 
-	return { abort }
-
+	setAbort(abort)
 }
 
 
 
-export const sendLLMMessage: SendLLMMessageFnTypeExternal = ({ messages, onText, onFinalMessage, onError, voidConfig }) => {
-	if (!voidConfig) return { abort: () => { } }
+export const sendLLMMessage: SendLLMMessageFnTypeExternal = ({ messages, onText, onFinalMessage, onError, voidConfig, setAbort }) => {
+	if (!voidConfig) return;
 
 	switch (voidConfig.default.whichApi) {
 		case 'anthropic':
-			return sendAnthropicMsg({ messages, onText, onFinalMessage, onError, voidConfig });
+			return sendAnthropicMsg({ messages, onText, onFinalMessage, onError, voidConfig, setAbort });
 		case 'openAI':
 		case 'openRouter':
 		case 'openAICompatible':
-			return sendOpenAIMsg({ messages, onText, onFinalMessage, onError, voidConfig });
+			return sendOpenAIMsg({ messages, onText, onFinalMessage, onError, voidConfig, setAbort });
 		case 'ollama':
-			return sendOllamaMsg({ messages, onText, onFinalMessage, onError, voidConfig });
+			return sendOllamaMsg({ messages, onText, onFinalMessage, onError, voidConfig, setAbort });
 		case 'greptile':
-			return sendGreptileMsg({ messages, onText, onFinalMessage, onError, voidConfig });
+			return sendGreptileMsg({ messages, onText, onFinalMessage, onError, voidConfig, setAbort });
 		default:
 			onError(`Error: whichApi was ${voidConfig.default.whichApi}, which is not recognized!`)
-			return { abort: () => { } }
 	}
 }

extensions/void/src/extension.ts

@@ -3,6 +3,8 @@ import { DisplayChangesProvider } from './DisplayChangesProvider';
 import { BaseDiffArea, ChatThreads, MessageFromSidebar, MessageToSidebar } from './shared_types';
 import { SidebarWebviewProvider } from './SidebarWebviewProvider';
 import { v4 as uuidv4 } from 'uuid'
+import { applyDiffLazily } from './common/ctrlL';
+import { getVoidConfig } from './sidebar/contextForConfig';
 
 const readFileContentOfUri = async (uri: vscode.Uri) => {
 	return Buffer.from(await vscode.workspace.fs.readFile(uri)).toString('utf8')
@@ -108,9 +110,19 @@ export function activate(context: vscode.ExtensionContext) {
 					// await vscode.workspace.applyEdit(workspaceEdit)
 					// await vscode.workspace.save(docUri)
 					// this._weAreEditing = false
-					await editor.edit(editBuilder => {
-						editBuilder.replace(new vscode.Range(diffArea.startLine, 0, diffArea.endLine, Number.MAX_SAFE_INTEGER), m.code);
-					});
+					const fileUri = editor.document.uri
+					const fileStr = await readFileContentOfUri(fileUri)
+					const voidConfig = getVoidConfig(context.globalState.get('partialVoidConfig') ?? {})
+
+					let abort = () => { } // TODO this is unused
+
+					// apply the change
+					applyDiffLazily({ fileUri, fileStr, diff: m.code, voidConfig, setAbort: (a) => { abort = a } })
+
+					// set the file equal to the change
+					// await editor.edit(editBuilder => {
+					// 	editBuilder.replace(new vscode.Range(diffArea.startLine, 0, diffArea.endLine, Number.MAX_SAFE_INTEGER), m.code);
+					// });
 
 					// rediff the changes based on the diffAreas
 					displayChangesProvider.refreshDiffAreas(editor.document.uri)
@@ -166,4 +178,3 @@ export function activate(context: vscode.ExtensionContext) {
 	// )
 
 }
-

extensions/void/src/sidebar/SidebarChat.tsx

@@ -163,7 +163,7 @@ export const SidebarChat = () => {
 		addMessageToHistory(newHistoryElt)
 
 		// send message to LLM
-		let { abort } = sendLLMMessage({
+		sendLLMMessage({
 			messages: [...(currentThread?.messages ?? []).map(m => ({ role: m.role, content: m.content })), { role: 'user', content: userContent }],
 			onText: (newText, fullText) => setMessageStream(fullText),
 			onFinalMessage: (content) => {
@@ -183,9 +183,12 @@ export const SidebarChat = () => {
 
 				setLatestError(error)
 			},
-			voidConfig: voidConfig
+			setAbort: (abort) => {
+				abortFnRef.current = abort
+			},
+			voidConfig,
 		})
-		abortFnRef.current = abort
+
 
 	}
 

extensions/void/src/sidebar/SidebarSettings.tsx

@@ -64,6 +64,10 @@ export const SidebarSettings = () => {
 					field='default'
 					param='whichApi'
 				/>
+				<SettingOfFieldAndParam
+					field='default'
+					param='maxTokens'
+				/>
 			</div>
 
 			<hr />

extensions/void/src/sidebar/contextForConfig.tsx

@@ -45,6 +45,18 @@ const voidConfigInfo: Record<
 			'anthropic',
 			configFields,
 		),
+
+		maxTokens: configEnum(
+			"Max number of tokens to output.",
+			'1024',
+			[
+				"1024",
+				"2048",
+				"4096",
+				"8192"
+			] as const,
+		),
+
 	},
 	anthropic: {
 		apikey: configString('Anthropic API key.', ''),
@@ -58,17 +70,6 @@ const voidConfigInfo: Record<
 				"claude-3-haiku-20240307"
 			] as const,
 		),
-
-		maxTokens: configEnum(
-			"Anthropic max number of tokens to output.",
-			'8192',
-			[
-				"1024",
-				"2048",
-				"4096",
-				"8192"
-			] as const,
-		),
 	},
 	openAI: {
 		apikey: configString('OpenAI API key.', ''),
@@ -270,7 +271,7 @@ export type VoidConfig = {
 
 
 
-const getVoidConfig = (currentConfig: PartialVoidConfig): VoidConfig => {
+export const getVoidConfig = (currentConfig: PartialVoidConfig): VoidConfig => {
 	const config = {} as PartialVoidConfig
 	for (let field of [...configFields, 'default'] as const) {
 		config[field] = {}