From 0ca2ad4930345973b92ca63a2e31f6e05baa5b7e Mon Sep 17 00:00:00 2001 From: mp Date: Sun, 17 Nov 2024 16:48:23 -0800 Subject: [PATCH] Create basic language server functionality --- extensions/void/package-lock.json | 69 +++- extensions/void/package.json | 7 +- .../LangaugeServer/createJsProgramGraph.ts | 333 ++++++++++++++++++ .../common/LangaugeServer/findFunctions.ts | 5 +- extensions/void/src/extension/extension.ts | 5 +- 5 files changed, 411 insertions(+), 8 deletions(-) create mode 100644 extensions/void/src/common/LangaugeServer/createJsProgramGraph.ts diff --git a/extensions/void/package-lock.json b/extensions/void/package-lock.json index d020b8ac..d8e80af1 100644 --- a/extensions/void/package-lock.json +++ b/extensions/void/package-lock.json @@ -8,7 +8,10 @@ "name": "void", "version": "0.0.1", "dependencies": { - "lru-cache": "^11.0.2" + "lru-cache": "^11.0.2", + "tree-sitter": "^0.21.1", + "tree-sitter-javascript": "^0.23.1", + "tree-sitter-python": "^0.23.4" }, "devDependencies": { "@anthropic-ai/sdk": "^0.29.2", @@ -6073,6 +6076,14 @@ "dev": true, "license": "MIT" }, + "node_modules/node-addon-api": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.2.2.tgz", + "integrity": "sha512-9emqXAKhVoNrQ792nLI/wpzPpJ/bj/YXxW0CvAau1+RdGBcCRF1Dmz7719zgVsQNrzHl9Tzn3ImZ4qWFarWL0A==", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -6114,6 +6125,16 @@ } } }, + "node_modules/node-gyp-build": { + "version": "4.8.3", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.3.tgz", + "integrity": "sha512-EMS95CMJzdoSKoIiXo8pxKoL8DYxwIZXYlLmgPb8KUv794abpnLK6ynsCAWNliOjREKruYKdzbh76HHYUHX7nw==", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-releases": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", @@ -8259,6 +8280,52 @@ "dev": true, "license": "MIT" }, + "node_modules/tree-sitter": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.21.1.tgz", + "integrity": "sha512-7dxoA6kYvtgWw80265MyqJlkRl4yawIjO7S5MigytjELkX43fV2WsAXzsNfO7sBpPPCF5Gp0+XzHk0DwLCq3xQ==", + "hasInstallScript": true, + "dependencies": { + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.0" + } + }, + "node_modules/tree-sitter-javascript": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.23.1.tgz", + "integrity": "sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA==", + "hasInstallScript": true, + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-python": { + "version": "0.23.4", + "resolved": "https://registry.npmjs.org/tree-sitter-python/-/tree-sitter-python-0.23.4.tgz", + "integrity": "sha512-MbmUAl7y5UCUWqHscHke7DdRDwQnVNMNKQYQc4Gq2p09j+fgPxaU8JVsuOI/0HD3BSEEe5k9j3xmdtIWbDtDgw==", + "hasInstallScript": true, + "dependencies": { + "node-addon-api": "^8.2.1", + "node-gyp-build": "^4.8.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, "node_modules/trim-lines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", diff --git a/extensions/void/package.json b/extensions/void/package.json index a3db23ec..4a8e3995 100644 --- a/extensions/void/package.json +++ b/extensions/void/package.json @@ -160,6 +160,9 @@ "uuid": "^10.0.0" }, "dependencies": { - "lru-cache": "^11.0.2" + "lru-cache": "^11.0.2", + "tree-sitter": "^0.21.1", + "tree-sitter-javascript": "^0.23.1", + "tree-sitter-python": "^0.23.4" } -} \ No newline at end of file +} diff --git a/extensions/void/src/common/LangaugeServer/createJsProgramGraph.ts b/extensions/void/src/common/LangaugeServer/createJsProgramGraph.ts new file mode 100644 index 00000000..4fb9bcaf --- /dev/null +++ b/extensions/void/src/common/LangaugeServer/createJsProgramGraph.ts @@ -0,0 +1,333 @@ +import * as vscode from 'vscode'; +import Parser from 'tree-sitter'; +import JavaScript from 'tree-sitter-javascript'; + +interface Definition { + file: string; + node: Parser.SyntaxNode; +} + +interface DefnUse { + parent: Parser.SyntaxNode; + file: string; +} + +interface ImportInfo { + source: string; + imported: string; +} + +class ProjectAnalyzer { + private parser: Parser; + private graph: Map>; + private visited: Set; + private parsedFiles: Map; + private imports: Map>; + private definitions: Map; + private fileStack: Set; + + constructor() { + this.parser = new Parser(); + this.parser.setLanguage(JavaScript); + this.graph = new Map(); + this.visited = new Set(); + this.parsedFiles = new Map(); + this.imports = new Map(); + this.definitions = new Map(); + this.fileStack = new Set(); + } + + async parseFile(filePath: string): Promise { + if (this.parsedFiles.has(filePath)) { + return this.parsedFiles.get(filePath)!; + } + + if (this.fileStack.has(filePath)) { + return null; // Circular import + } + + this.fileStack.add(filePath); + + try { + const uri = vscode.Uri.file(filePath); + const document = await vscode.workspace.openTextDocument(uri); + const code = document.getText(); + const tree = this.parser.parse(code); + + this.parsedFiles.set(filePath, tree); + this.collectImports(filePath, tree); + this.collectDefinitions(filePath, tree); + + return tree; + } catch (error) { + console.error(`Error parsing ${filePath}:`, error); + return null; + } finally { + this.fileStack.delete(filePath); + } + } + + private collectImports(filePath: string, tree: Parser.Tree): void { + const fileImports = new Map(); + + const visit = (node: Parser.SyntaxNode): void => { + if (node.type === 'import_declaration') { + const source = node.childForFieldName('source')?.text.slice(1, -1) ?? ''; + const specifiers = node.childForFieldName('specifiers'); + + specifiers?.children.forEach(spec => { + if (spec.type === 'import_specifier') { + const local = spec.childForFieldName('local')?.text ?? ''; + const imported = spec.childForFieldName('imported')?.text ?? ''; + fileImports.set(local, { source, imported }); + } + }); + } + node.children.forEach(visit); + }; + + visit(tree.rootNode); + this.imports.set(filePath, fileImports); + } + + private collectDefinitions(filePath: string, tree: Parser.Tree): void { + const visit = (node: Parser.SyntaxNode): void => { + if (node.type === 'function_declaration') { + const name = node.childForFieldName('name')?.text ?? ''; + this.definitions.set(name, { file: filePath, node }); + } + else if (node.type === 'variable_declarator') { + const name = node.childForFieldName('name')?.text; + const value = node.childForFieldName('value'); + if (name && (value?.type === 'arrow_function' || value?.type === 'function')) { + this.definitions.set(name, { file: filePath, node: value }); + } + } + node.children.forEach(visit); + }; + + visit(tree.rootNode); + } + + private async getTypeFromPosition(uri: vscode.Uri, position: vscode.Position): Promise { + const hover = await vscode.commands.executeCommand( + 'vscode.executeHoverProvider', + uri, + position + ); + + if (hover?.[0]?.contents.length) { + for (const content of hover[0].contents) { + let hoverText = typeof content === 'string' ? + content : + ('value' in content ? content.value : ''); + + // Remove typescript backticks if present + hoverText = hoverText.replace(/```typescript\s*/, '').replace(/```\s*$/, ''); + console.log('Processing hover text:', hoverText); + + // Extract the type information - look for the type after the colon + const typeMatches = [ + /:\s*([\w<>]+)(?:\[\])?/, // matches "foo: Type" or "foo: Type[]" + /var\s+\w+:\s*([\w<>]+)/, // matches "var foo: Type" + /\(type\)\s+[\w<>]+:\s*([\w<>]+)/, // matches "(type) foo: Type" + /\(method\)\s*([\w<>]+)\./ // matches "(method) Type.method" + ]; + + for (const pattern of typeMatches) { + const match = pattern.exec(hoverText); + if (match) { + let type = match[1]; + // Handle array types + if (hoverText.includes('[]')) { + return 'Array'; + } + // Extract base type from generics + if (type.includes('<')) { + type = type.split('<')[0]; + } + return type; + } + } + } + } + return null; + } + + private async getCallsInDefn(defnNode: Parser.SyntaxNode, currentFile: string): Promise> { + const calls = new Set(); + const fileImports = this.imports.get(currentFile) ?? new Map(); + const uri = vscode.Uri.file(currentFile); + + const visit = async (node: Parser.SyntaxNode): Promise => { + if (node.type === 'call_expression') { + const callee = node.childForFieldName('function'); + if (callee?.type === 'identifier') { + const name = callee.text; + const importInfo = fileImports.get(name); + if (importInfo) { + calls.add(`${importInfo.source}:${importInfo.imported}`); + } else { + calls.add(name); + } + } + else if (callee?.type === 'member_expression') { + const method = callee.childForFieldName('property')?.text; + const object = callee.childForFieldName('object'); + + if (method && object) { + const position = new vscode.Position( + object.startPosition.row, + object.startPosition.column + ); + + const type = await this.getTypeFromPosition(uri, position); + if (type) { + calls.add(`${type}.${method}`); + } else { + calls.add(`method:${method}`); + } + } + } + } + + for (const child of node.children) { + await visit(child); + } + }; + + await visit(defnNode); + return calls; + } + + private gotoDefn(name: string): Definition | null { + if (name.includes(':')) { + const [file, funcName] = name.split(':'); + const def = this.definitions.get(funcName); + return def ?? null; + } + + return this.definitions.get(name) ?? null; + } + + private getUses(defnNode: Parser.SyntaxNode, currentFile: string): DefnUse[] { + const uses: DefnUse[] = []; + + let fnName: string | undefined; + if (defnNode.type === 'function_declaration') { + fnName = defnNode.childForFieldName('name')?.text; + } else if (defnNode.type === 'arrow_function' || defnNode.type === 'function') { + const parent = defnNode.parent; + if (parent?.type === 'variable_declarator') { + fnName = parent.childForFieldName('name')?.text; + } + } + + if (!fnName) return uses; + + for (const [file, tree] of this.parsedFiles) { + const visit = (node: Parser.SyntaxNode): void => { + if (node.type === 'call_expression') { + const callee = node.childForFieldName('function'); + if (callee?.type === 'identifier' && callee.text === fnName) { + let current: Parser.SyntaxNode | null = node; + while (current) { + if (current.type === 'function_declaration' || + current.type === 'arrow_function' || + current.type === 'function') { + uses.push({ parent: current, file }); + break; + } + current = current.parent; + } + } + } + node.children.forEach(visit); + }; + + visit(tree.rootNode); + } + + return uses; + } + + private async visitAllNodesInGraphFromDefinition(defn: Parser.SyntaxNode, currentFile: string): Promise { + let defnName: string | undefined; + if (defn.type === 'function_declaration') { + defnName = defn.childForFieldName('name')?.text; + } else if (defn.type === 'arrow_function' || defn.type === 'function') { + const parent = defn.parent; + if (parent?.type === 'variable_declarator') { + defnName = parent.childForFieldName('name')?.text; + } + } + + if (!defnName) return; + + const fullName = `${currentFile}:${defnName}`; + if (this.visited.has(fullName)) return; + + const calls = await this.getCallsInDefn(defn, currentFile); + this.graph.set(fullName, calls); + this.visited.add(fullName); + + const callDefns = Array.from(calls).map(call => this.gotoDefn(call)); + for (const callDefn of callDefns) { + if (callDefn) { + await this.visitAllNodesInGraphFromDefinition(callDefn.node, callDefn.file); + } + } + + const defnUses = this.getUses(defn, currentFile); + for (const defnUse of defnUses) { + await this.visitAllNodesInGraphFromDefinition(defnUse.parent, defnUse.file); + } + } + + async analyze(entryFile: string): Promise>> { + const tree = await this.parseFile(entryFile); + if (!tree) return new Map(); + + const visit = async (node: Parser.SyntaxNode): Promise => { + if (node.type === 'function_declaration') { + await this.visitAllNodesInGraphFromDefinition(node, entryFile); + } + else if (node.type === 'variable_declarator') { + const value = node.childForFieldName('value'); + if (value?.type === 'arrow_function' || value?.type === 'function') { + await this.visitAllNodesInGraphFromDefinition(value, entryFile); + } + } + for (const child of node.children) { + await visit(child); + } + }; + + await visit(tree.rootNode); + return this.graph; + } +} + +export async function runTreeSitter(filePath?: string): Promise> | null> { + const editor = vscode.window.activeTextEditor; + if (!editor && !filePath) { + vscode.window.showWarningMessage('No active editor found'); + return null; + } + + try { + const targetPath = filePath ?? editor!.document.uri.fsPath; + const analyzer = new ProjectAnalyzer(); + const graph = await analyzer.analyze(targetPath); + + for (const [defn, calls] of graph) { + console.log(`${defn} calls: ${[...calls].join(', ')}`); + } + + return graph; + } catch (error) { + console.error('Error analyzing file:', error); + vscode.window.showErrorMessage('Error analyzing file'); + return null; + } +} \ No newline at end of file diff --git a/extensions/void/src/common/LangaugeServer/findFunctions.ts b/extensions/void/src/common/LangaugeServer/findFunctions.ts index dafc0287..570b4369 100644 --- a/extensions/void/src/common/LangaugeServer/findFunctions.ts +++ b/extensions/void/src/common/LangaugeServer/findFunctions.ts @@ -2,7 +2,7 @@ import * as vscode from 'vscode'; const legend = new vscode.SemanticTokensLegend([], []); -export async function getFunctionTokens() { +export async function findFunctions() { const editor = vscode.window.activeTextEditor; if (!editor) return; @@ -20,7 +20,6 @@ export async function getFunctionTokens() { const allTokens = decodeTokens(tokens, document); - console.log("Tokens:", allTokens); return allTokens; } @@ -60,4 +59,4 @@ function decodeTokens(tokens: vscode.SemanticTokens, document: vscode.TextDocume } return decodedTokens; -} \ No newline at end of file +} diff --git a/extensions/void/src/extension/extension.ts b/extensions/void/src/extension/extension.ts index 5ac18db3..82de53b9 100644 --- a/extensions/void/src/extension/extension.ts +++ b/extensions/void/src/extension/extension.ts @@ -10,7 +10,7 @@ import { readFileContentOfUri } from './extensionLib/readFileContentOfUri'; import { SidebarWebviewProvider } from './providers/SidebarWebviewProvider'; import { CtrlKWebviewProvider } from './providers/CtrlKWebviewProvider'; import { AutocompleteProvider } from './AutcompleteProvider'; -import { getFunctionTokens } from '../common/LangaugeServer/findFunctions'; +import { runTreeSitter } from '../common/LangaugeServer/createJsProgramGraph'; // // this comes from vscode.proposed.editorInsets.d.ts // declare module 'vscode' { @@ -193,7 +193,8 @@ export function activate(context: vscode.ExtensionContext) { // 7. Language Server - let disposable = vscode.commands.registerCommand('typeInspector.inspect', getFunctionTokens); + console.log('run lsp') + let disposable = vscode.commands.registerCommand('typeInspector.inspect', runTreeSitter); context.subscriptions.push(disposable);