From 0f2a3361fd7210613bb8213aec7a6d4d1870f95b Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Thu, 21 Dec 2023 17:14:22 -0500 Subject: [PATCH 001/119] Create ProjectManager --- src/KeyedThrottler.spec.ts | 24 +++ src/LanguageServer.spec.ts | 48 ++--- src/LanguageServer.ts | 335 ++++++--------------------------- src/ProgramBuilder.ts | 8 +- src/lsp/DocumentManager.ts | 68 +++++++ src/lsp/Project.ts | 22 +++ src/lsp/ProjectManager.spec.ts | 85 +++++++++ src/lsp/ProjectManager.ts | 321 +++++++++++++++++++++++++++++++ 8 files changed, 605 insertions(+), 306 deletions(-) create mode 100644 src/KeyedThrottler.spec.ts create mode 100644 src/lsp/DocumentManager.ts create mode 100644 src/lsp/Project.ts create mode 100644 src/lsp/ProjectManager.spec.ts create mode 100644 src/lsp/ProjectManager.ts diff --git a/src/KeyedThrottler.spec.ts b/src/KeyedThrottler.spec.ts new file mode 100644 index 000000000..ff729c20d --- /dev/null +++ b/src/KeyedThrottler.spec.ts @@ -0,0 +1,24 @@ +import { KeyedThrottler } from './KeyedThrottler'; +import { expect } from './chai-config.spec'; + +describe('KeyedThrottler', () => { + let throttler: KeyedThrottler; + beforeEach(() => { + throttler = new KeyedThrottler(0); + }); + + it('returns the correct value for each resolved promise', async () => { + let results = [null, null, null, null, null]; + + //should only run index 0 and index 4 + let promises = [0, 1, 2, 3, 4].map(x => { + return throttler.run('same-key', () => { + results[x] = x; + }); + }); + await Promise.all(promises); + expect( + results + ).to.eql([0, null, null, null, 4]); + }); +}); diff --git a/src/LanguageServer.spec.ts b/src/LanguageServer.spec.ts index 6b69a899a..ce3f56cca 100644 --- a/src/LanguageServer.spec.ts +++ b/src/LanguageServer.spec.ts @@ -174,7 +174,7 @@ describe('LanguageServer', () => { firstRunPromise: deferred.promise }; //make a new not-completed project - server.projects.push(project); + server['projectManager'].projects.push(project); //this call should wait for the builder to finish let p = server['sendDiagnostics'](); @@ -190,7 +190,7 @@ describe('LanguageServer', () => { }); it('dedupes diagnostics found at same location from multiple projects', async () => { - server.projects.push({ + server['projectManager'].projects.push({ firstRunPromise: Promise.resolve(), builder: { getDiagnostics: () => { @@ -230,7 +230,7 @@ describe('LanguageServer', () => { const promise = new Promise((resolve) => { stub = sinon.stub(connection, 'sendDiagnostics').callsFake(resolve as any); }); - const { program } = server.projects[0].builder; + const { program } = server['projectManager'].projects[0].builder; program.setFile('source/lib.bs', ` sub lib() functionDoesNotExist() @@ -261,12 +261,12 @@ describe('LanguageServer', () => { } as any); writeToFs(mainPath, `sub main(): return: end sub`); await server['onInitialized'](); - expect(server.projects[0].builder.program.hasFile(mainPath)).to.be.true; + expect(server['projectManager'].projects[0].builder.program.hasFile(mainPath)).to.be.true; //move a file into the directory...the program should detect it let libPath = s`${workspacePath}/source/lib.brs`; writeToFs(libPath, 'sub lib(): return : end sub'); - server.projects[0].configFilePath = `${workspacePath}/bsconfig.json`; + server['projectManager'].projects[0].configFilePath = `${workspacePath}/bsconfig.json`; await server['onDidChangeWatchedFiles']({ changes: [{ uri: getFileProtocolPath(libPath), @@ -282,7 +282,7 @@ describe('LanguageServer', () => { // } ] }); - expect(server.projects[0].builder.program.hasFile(libPath)).to.be.true; + expect(server['projectManager'].projects[0].builder.program.hasFile(libPath)).to.be.true; }); }); @@ -333,7 +333,7 @@ describe('LanguageServer', () => { it('loads workspace as project', async () => { server.run(); - expect(server.projects).to.be.lengthOf(0); + expect(server['projectManager'].projects).to.be.lengthOf(0); fsExtra.ensureDirSync(workspacePath); @@ -341,7 +341,7 @@ describe('LanguageServer', () => { //no child bsconfig.json files, use the workspacePath expect( - server.projects.map(x => x.projectPath) + server['projectManager'].projects.map(x => x.projectPath) ).to.eql([ workspacePath ]); @@ -353,7 +353,7 @@ describe('LanguageServer', () => { //2 child bsconfig.json files. Use those folders as projects, and don't use workspacePath expect( - server.projects.map(x => x.projectPath).sort() + server['projectManager'].projects.map(x => x.projectPath).sort() ).to.eql([ s`${workspacePath}/project1`, s`${workspacePath}/project2` @@ -364,7 +364,7 @@ describe('LanguageServer', () => { //1 child bsconfig.json file. Still don't use workspacePath expect( - server.projects.map(x => x.projectPath) + server['projectManager'].projects.map(x => x.projectPath) ).to.eql([ s`${workspacePath}/project1` ]); @@ -374,7 +374,7 @@ describe('LanguageServer', () => { //back to no child bsconfig.json files. use workspacePath again expect( - server.projects.map(x => x.projectPath) + server['projectManager'].projects.map(x => x.projectPath) ).to.eql([ workspacePath ]); @@ -395,7 +395,7 @@ describe('LanguageServer', () => { //no child bsconfig.json files, use the workspacePath expect( - server.projects.map(x => x.projectPath) + server['projectManager'].projects.map(x => x.projectPath) ).to.eql([ workspacePath ]); @@ -414,7 +414,7 @@ describe('LanguageServer', () => { await server['syncProjects'](); expect( - server.projects.map(x => x.projectPath).sort() + server['projectManager'].projects.map(x => x.projectPath).sort() ).to.eql([ s`${tempDir}/root`, s`${tempDir}/root/subdir` @@ -439,7 +439,7 @@ describe('LanguageServer', () => { await server['syncProjects'](); expect( - server.projects.map(x => x.projectPath).sort() + server['projectManager'].projects.map(x => x.projectPath).sort() ).to.eql([ s`${tempDir}/project1`, s`${tempDir}/sub/dir/project2` @@ -456,7 +456,7 @@ describe('LanguageServer', () => { fsExtra.outputFileSync(s`${rootDir}/source/lib.brs`, ''); await server['syncProjects'](); - const stub2 = sinon.stub(server.projects[0].builder.program, 'setFile'); + const stub2 = sinon.stub(server['projectManager'].projects[0].builder.program, 'setFile'); await server['onDidChangeWatchedFiles']({ changes: [{ @@ -481,7 +481,7 @@ describe('LanguageServer', () => { fsExtra.outputFileSync(s`${externalDir}/source/lib.brs`, ''); await server['syncProjects'](); - const stub2 = sinon.stub(server.projects[0].builder.program, 'setFile'); + const stub2 = sinon.stub(server['projectManager'].projects[0].builder.program, 'setFile'); await server['onDidChangeWatchedFiles']({ changes: [{ @@ -503,7 +503,7 @@ describe('LanguageServer', () => { beforeEach(async () => { server['connection'] = server['createConnection'](); await server['createProject'](workspacePath); - program = server.projects[0].builder.program; + program = server['projectManager'].projects[0].builder.program; const name = `CallComponent`; callDocument = addScriptFile(name, ` @@ -594,7 +594,7 @@ describe('LanguageServer', () => { beforeEach(async () => { server['connection'] = server['createConnection'](); await server['createProject'](workspacePath); - program = server.projects[0].builder.program; + program = server['projectManager'].projects[0].builder.program; const functionFileBaseName = 'buildAwesome'; functionDocument = addScriptFile(functionFileBaseName, ` @@ -662,7 +662,7 @@ describe('LanguageServer', () => { beforeEach(async () => { server['connection'] = server['createConnection'](); await server['createProject'](workspacePath); - program = server.projects[0].builder.program; + program = server['projectManager'].projects[0].builder.program; const functionFileBaseName = 'buildAwesome'; functionDocument = addScriptFile(functionFileBaseName, ` @@ -787,7 +787,7 @@ describe('LanguageServer', () => { beforeEach(async () => { server['connection'] = server['createConnection'](); await server['createProject'](workspacePath); - program = server.projects[0].builder.program; + program = server['projectManager'].projects[0].builder.program; }); it('should return the expected symbols even if pulled from cache', async () => { @@ -876,7 +876,7 @@ describe('LanguageServer', () => { beforeEach(async () => { server['connection'] = server['createConnection'](); await server['createProject'](workspacePath); - program = server.projects[0].builder.program; + program = server['projectManager'].projects[0].builder.program; }); it('should return the expected symbols even if pulled from cache', async () => { @@ -1072,7 +1072,7 @@ describe('LanguageServer', () => { await server['syncProjects'](); const afterSpy = sinon.spy(); //make a plugin that changes string text - server.projects[0].builder.program.plugins.add({ + server['projectManager'].projects[0].builder.program.plugins.add({ name: 'test-plugin', beforeProgramTranspile: (program, entries, editor) => { const file = program.getFile('source/main.bs'); @@ -1133,7 +1133,7 @@ describe('LanguageServer', () => { fsExtra.outputFileSync(s`${rootDir}/source/sgnode.bs`, getContents()); server.run(); await server['syncProjects'](); - expectZeroDiagnostics(server.projects[0].builder.program); + expectZeroDiagnostics(server['projectManager'].projects[0].builder.program); fsExtra.outputFileSync(s`${rootDir}/source/sgnode.bs`, getContents()); const changeWatchedFilesPromise = server['onDidChangeWatchedFiles']({ @@ -1154,7 +1154,7 @@ describe('LanguageServer', () => { changeWatchedFilesPromise, semanticTokensPromise ]); - expectZeroDiagnostics(server.projects[0].builder.program); + expectZeroDiagnostics(server['projectManager'].projects[0].builder.program); }); }); diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 3035709a2..f721d9fa0 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -37,22 +37,35 @@ import { URI } from 'vscode-uri'; import { TextDocument } from 'vscode-languageserver-textdocument'; import type { BsConfig } from './BsConfig'; import { Deferred } from './deferred'; -import { DiagnosticMessages } from './DiagnosticMessages'; import { ProgramBuilder } from './ProgramBuilder'; import { standardizePath as s, util } from './util'; import { Logger } from './Logger'; import { Throttler } from './Throttler'; -import { KeyedThrottler } from './KeyedThrottler'; import { DiagnosticCollection } from './DiagnosticCollection'; import { isBrsFile } from './astUtils/reflection'; import { encodeSemanticTokens, semanticTokensLegend } from './SemanticTokenUtils'; import type { BusyStatus } from './BusyStatusTracker'; import { BusyStatusTracker } from './BusyStatusTracker'; +import { ProjectManager } from './lsp/ProjectManager'; export class LanguageServer { + constructor() { + this.projectManager = new ProjectManager(); + //anytime a project finishes validation, send diagnostics + this.projectManager.on('flush-diagnostics', () => { + void this.sendDiagnostics(); + }); + //allow the lsp to provide file contents + this.projectManager.addFileResolver(this.documentFileResolver.bind(this)); + } + private connection = undefined as Connection; - public projects = [] as Project[]; + /** + * Manages all projects for this language server + */ + private projectManager: ProjectManager; + /** * The number of milliseconds that should be used for language server typing debouncing @@ -86,8 +99,6 @@ export class LanguageServer { private loggerSubscription: () => void; - private keyedThrottler = new KeyedThrottler(this.debounceTimeout); - public validateThrottler = new Throttler(0); private sendDiagnosticsThrottler = new Throttler(0); @@ -251,17 +262,7 @@ export class LanguageServer { * Ask the client for the list of `files.exclude` patterns. Useful when determining if we should process a file */ private async getWorkspaceExcludeGlobs(workspaceFolder: string): Promise { - let config = { - exclude: {} as Record - }; - //if supported, ask vscode for the `files.exclude` configuration - if (this.hasConfigurationCapability) { - //get any `files.exclude` globs to use to filter - config = await this.connection.workspace.getConfiguration({ - scopeUri: workspaceFolder, - section: 'files' - }); - } + const config = await this.getClientConfiguration(workspaceFolder, 'files'); return Object .keys(config?.exclude ?? {}) .filter(x => config?.exclude?.[x]) @@ -280,51 +281,6 @@ export class LanguageServer { ]); } - /** - * Scan the workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. - * If none are found, then the workspaceFolder itself is treated as a project - */ - @TrackBusyStatus - private async getProjectPaths(workspaceFolder: string) { - const excludes = (await this.getWorkspaceExcludeGlobs(workspaceFolder)).map(x => s`!${x}`); - const files = await rokuDeploy.getFilePaths([ - '**/bsconfig.json', - //exclude all files found in `files.exclude` - ...excludes - ], workspaceFolder); - //if we found at least one bsconfig.json, then ALL projects must have a bsconfig.json. - if (files.length > 0) { - return files.map(file => s`${path.dirname(file.src)}`); - } - - //look for roku project folders - const rokuLikeDirs = (await Promise.all( - //find all folders containing a `manifest` file - (await rokuDeploy.getFilePaths([ - '**/manifest', - ...excludes - - //is there at least one .bs|.brs file under the `/source` folder? - ], workspaceFolder)).map(async manifestEntry => { - const manifestDir = path.dirname(manifestEntry.src); - const files = await rokuDeploy.getFilePaths([ - 'source/**/*.{brs,bs}', - ...excludes - ], manifestDir); - if (files.length > 0) { - return manifestDir; - } - }) - //throw out nulls - )).filter(x => !!x); - if (rokuLikeDirs.length > 0) { - return rokuLikeDirs; - } - - //treat the workspace folder as a brightscript project itself - return [workspaceFolder]; - } - /** * Find all folders with bsconfig.json files in them, and treat each as a project. * Treat workspaces that don't have a bsconfig.json as a project. @@ -333,54 +289,47 @@ export class LanguageServer { */ @TrackBusyStatus private async syncProjects() { - const workspacePaths = await this.getWorkspacePaths(); - let projectPaths = (await Promise.all( - workspacePaths.map(async workspacePath => { - const projectPaths = await this.getProjectPaths(workspacePath); - return projectPaths.map(projectPath => ({ - projectPath: projectPath, - workspacePath: workspacePath - })); + // get all workspace paths from the client + let workspaces = await Promise.all( + (await this.connection.workspace.getWorkspaceFolders() ?? []).map(async (x) => { + const workspaceFolder = util.uriToPath(x.uri); + return { + workspaceFolder: workspaceFolder, + excludePatterns: await this.getWorkspaceExcludeGlobs(workspaceFolder), + bsconfigPath: (await this.getClientConfiguration(x.uri, 'brightscript'))?.configFile + }; }) - )).flat(1); - - //delete projects not represented in the list - for (const project of this.getProjects()) { - if (!projectPaths.find(x => x.projectPath === project.projectPath)) { - this.removeProject(project); - } - } + ); - //exclude paths to projects we already have - projectPaths = projectPaths.filter(x => { - //only keep this project path if there's not a project with that path - return !this.projects.find(project => project.projectPath === x.projectPath); - }); - //dedupe by project path - projectPaths = [ - ...projectPaths.reduce( - (acc, x) => acc.set(x.projectPath, x), - new Map() - ).values() - ]; + await this.projectManager.syncProjects(workspaces); - //create missing projects - await Promise.all( - projectPaths.map(x => this.createProject(x.projectPath, x.workspacePath)) - ); //flush diagnostics await this.sendDiagnostics(); } /** - * Get all workspace paths from the client + * Given a workspaceFolder path, get the specified configuration from the client (if applicable). + * Be sure to use optional chaining to traverse the result in case that configuration doesn't exist or the client doesn't support `getConfiguration` + * @param workspaceFolder the folder for the workspace in the client */ - private async getWorkspacePaths() { - let workspaceFolders = await this.connection.workspace.getWorkspaceFolders() ?? []; - return workspaceFolders.map((x) => { - return util.uriToPath(x.uri); - }); + private async getClientConfiguration>(workspaceFolder: string, section: string): Promise { + let scopeUri: string; + if (workspaceFolder.startsWith('file:')) { + scopeUri = URI.parse(workspaceFolder).toString(); + } else { + scopeUri = URI.file(workspaceFolder).toString(); + } + let config = {}; + + //if the client supports configuration, look for config group called "brightscript" + if (this.hasConfigurationCapability) { + config = await this.connection.workspace.getConfiguration({ + scopeUri: scopeUri, + section: section + }); + } + return config as T; } /** @@ -461,147 +410,6 @@ export class LanguageServer { } } - private async getConfigFilePath(workspacePath: string) { - let scopeUri: string; - if (workspacePath.startsWith('file:')) { - scopeUri = URI.parse(workspacePath).toString(); - } else { - scopeUri = URI.file(workspacePath).toString(); - } - let config = { - configFile: undefined - }; - //if the client supports configuration, look for config group called "brightscript" - if (this.hasConfigurationCapability) { - config = await this.connection.workspace.getConfiguration({ - scopeUri: scopeUri, - section: 'brightscript' - }); - } - let configFilePath: string; - - //if there's a setting, we need to find the file or show error if it can't be found - if (config?.configFile) { - configFilePath = path.resolve(workspacePath, config.configFile); - if (await util.pathExists(configFilePath)) { - return configFilePath; - } else { - this.sendCriticalFailure(`Cannot find config file specified in user / workspace settings at '${configFilePath}'`); - } - } - - //default to config file path found in the root of the workspace - configFilePath = path.resolve(workspacePath, 'bsconfig.json'); - if (await util.pathExists(configFilePath)) { - return configFilePath; - } - - //look for the deprecated `brsconfig.json` file - configFilePath = path.resolve(workspacePath, 'brsconfig.json'); - if (await util.pathExists(configFilePath)) { - return configFilePath; - } - - //no config file could be found - return undefined; - } - - - /** - * A unique project counter to help distinguish log entries in lsp mode - */ - private projectCounter = 0; - - /** - * @param projectPath path to the project - * @param workspacePath path to the workspace in which all project should reside or are referenced by - * @param projectNumber an optional project number to assign to the project. Used when reloading projects that should keep the same number - */ - @TrackBusyStatus - private async createProject(projectPath: string, workspacePath = projectPath, projectNumber?: number) { - workspacePath ??= projectPath; - let project = this.projects.find((x) => x.projectPath === projectPath); - //skip this project if we already have it - if (project) { - return; - } - - let builder = new ProgramBuilder(); - projectNumber ??= this.projectCounter++; - builder.logger.prefix = `[prj${projectNumber}]`; - builder.logger.log(`Created project #${projectNumber} for: "${projectPath}"`); - - //flush diagnostics every time the program finishes validating - builder.plugins.add({ - name: 'bsc-language-server', - afterProgramValidate: () => { - void this.sendDiagnostics(); - } - }); - - //prevent clearing the console on run...this isn't the CLI so we want to keep a full log of everything - builder.allowConsoleClearing = false; - - //look for files in our in-memory cache before going to the file system - builder.addFileResolver(this.documentFileResolver.bind(this)); - - let configFilePath = await this.getConfigFilePath(projectPath); - - let cwd = projectPath; - - //if the config file exists, use it and its folder as cwd - if (configFilePath && await util.pathExists(configFilePath)) { - cwd = path.dirname(configFilePath); - } else { - //config file doesn't exist...let `brighterscript` resolve the default way - configFilePath = undefined; - } - - const firstRunDeferred = new Deferred(); - - let newProject: Project = { - projectNumber: projectNumber, - builder: builder, - firstRunPromise: firstRunDeferred.promise, - projectPath: projectPath, - workspacePath: workspacePath, - isFirstRunComplete: false, - isFirstRunSuccessful: false, - configFilePath: configFilePath, - isStandaloneFileProject: false - }; - - this.projects.push(newProject); - - try { - await builder.run({ - cwd: cwd, - project: configFilePath, - watch: false, - createPackage: false, - deploy: false, - copyToStaging: false, - showDiagnosticsInConsole: false - }); - newProject.isFirstRunComplete = true; - newProject.isFirstRunSuccessful = true; - firstRunDeferred.resolve(); - } catch (e) { - builder.logger.error(e); - firstRunDeferred.reject(e); - newProject.isFirstRunComplete = true; - newProject.isFirstRunSuccessful = false; - } - //if we found a deprecated brsconfig.json, add a diagnostic warning the user - if (configFilePath && path.basename(configFilePath) === 'brsconfig.json') { - builder.addDiagnostic(configFilePath, { - ...DiagnosticMessages.brsConfigJsonIsDeprecated(), - range: util.createRange(0, 0, 0, 0) - }); - return this.sendDiagnostics(); - } - } - private async createStandaloneFileProject(srcPath: string) { //skip this workspace if we already have it if (this.standaloneFileProjects[srcPath]) { @@ -692,7 +500,7 @@ export class LanguageServer { let filePath = util.uriToPath(params.textDocument.uri); //wait until the file has settled - await this.keyedThrottler.onIdleOnce(filePath, true); + await this.onValidateSettled(); let completions = this .getProjects() @@ -729,7 +537,7 @@ export class LanguageServer { let srcPath = util.uriToPath(params.textDocument.uri); //wait until the file has settled - await this.keyedThrottler.onIdleOnce(srcPath, true); + await this.onValidateSettled(); const codeActions = this .getProjects() @@ -746,17 +554,6 @@ export class LanguageServer { return codeActions; } - /** - * Remove a project from the language server - */ - private removeProject(project: Project) { - const idx = this.projects.indexOf(project); - if (idx > -1) { - this.projects.splice(idx, 1); - } - project?.builder?.dispose(); - } - /** * Reload each of the specified workspaces */ @@ -967,9 +764,7 @@ export class LanguageServer { //All functions below can handle being given a file path AND a folder path, and will only operate on the one they are looking for let consumeCount = 0; await Promise.all(changes.map(async (change) => { - await this.keyedThrottler.run(change.srcPath, async () => { - consumeCount += await this.handleFileChange(project, change) ? 1 : 0; - }); + consumeCount += await this.handleFileChange(project, change) ? 1 : 0; })); if (consumeCount > 0) { @@ -1100,7 +895,7 @@ export class LanguageServer { try { //throttle file processing. first call is run immediately, and then the last call is processed. - await this.keyedThrottler.run(filePath, () => { + await this.keyedThrottler.run(filePath, async () => { let documentText = document.getText(); for (const project of this.getProjects()) { @@ -1115,9 +910,9 @@ export class LanguageServer { }, documentText); } } + // validate all projects + await this.validateAllThrottled(); }); - // validate all projects - await this.validateAllThrottled(); } catch (e: any) { this.sendCriticalFailure(`Critical error parsing/validating ${filePath}: ${e.message}`); } @@ -1283,6 +1078,7 @@ export class LanguageServer { private async sendDiagnostics() { await this.sendDiagnosticsThrottler.run(async () => { + // await this.projectManager.onSettle(); //wait for all programs to finish running. This ensures the `Program` exists. await Promise.all( this.projects.map(x => x.firstRunPromise) @@ -1332,27 +1128,6 @@ export class LanguageServer { } } -export interface Project { - /** - * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging - */ - projectNumber: number; - firstRunPromise: Promise; - builder: ProgramBuilder; - /** - * The path to where the project resides - */ - projectPath: string; - /** - * The path to the workspace where this project resides. A workspace can have multiple projects (by adding a bsconfig.json to each folder). - */ - workspacePath: string; - isFirstRunComplete: boolean; - isFirstRunSuccessful: boolean; - configFilePath?: string; - isStandaloneFileProject: boolean; -} - export enum CustomCommands { TranspileFile = 'TranspileFile' } diff --git a/src/ProgramBuilder.ts b/src/ProgramBuilder.ts index 6afee8800..5cdf5efcc 100644 --- a/src/ProgramBuilder.ts +++ b/src/ProgramBuilder.ts @@ -41,8 +41,12 @@ export class ProgramBuilder { public plugins: PluginInterface = new PluginInterface([], { logger: this.logger }); private fileResolvers = [] as FileResolver[]; - public addFileResolver(fileResolver: FileResolver) { - this.fileResolvers.push(fileResolver); + /** + * Add file resolvers that will be able to provide file contents before loading from the file system + * @param fileResolvers + */ + public addFileResolver(...fileResolvers: FileResolver[]) { + this.fileResolvers.push(...fileResolvers); } /** diff --git a/src/lsp/DocumentManager.ts b/src/lsp/DocumentManager.ts new file mode 100644 index 000000000..744a564fe --- /dev/null +++ b/src/lsp/DocumentManager.ts @@ -0,0 +1,68 @@ +/** + * Maintains a queued/buffered list of file operations. These operations don't actually do anything on their own. + * You need to call the .apply() function and provide an action to operate on them. + */ +export class DocumentManager { + private queue = new Map(); + + /** + * Add/set the contents of a file + * @param document + */ + public set(document: Document) { + if (this.queue.has(document.paths.src)) { + this.queue.delete(document.paths.src); + } + this.queue.set(document.paths.src, { action: 'set', document: document }); + } + + /** + * Delete a file + * @param document + */ + public delete(document: Document) { + if (this.queue.has(document.paths.src)) { + this.queue.delete(document.paths.src); + } + this.queue.set(document.paths.src, { action: 'delete', document: document }); + } + + /** + * Are there any pending documents that need to be flushed + */ + public get hasPendingChanges() { + return this.queue.size > 0; + } + + /** + * Indicates whether we are currently in the middle of an `apply()` session or not + */ + public isBlocked = false; + + /** + * Get all of the pending documents and clear the queue + */ + public async apply(action: (actions: DocumentAction[]) => any): Promise { + this.isBlocked = true; + try { + const documentActions = [...this.queue.values()]; + const result = await Promise.resolve(action(documentActions)); + return result; + } finally { + this.isBlocked = false; + } + } +} + +export interface DocumentAction { + action: 'set' | 'delete'; + document: Document; +} + +export interface Document { + paths: { + src: string; + dest: string; + }; + getText: () => string; +} diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts new file mode 100644 index 000000000..42a792b9a --- /dev/null +++ b/src/lsp/Project.ts @@ -0,0 +1,22 @@ +import type { ProgramBuilder } from '../ProgramBuilder'; + +export interface Project { + /** + * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging + */ + projectNumber: number; + firstRunPromise: Promise; + builder: ProgramBuilder; + /** + * The path to where the project resides + */ + projectPath: string; + /** + * The path to the workspace where this project resides. A workspace can have multiple projects (by adding a bsconfig.json to each folder). + */ + workspaceFolder: string; + isFirstRunComplete: boolean; + isFirstRunSuccessful: boolean; + configFilePath?: string; + isStandaloneFileProject: boolean; +} diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts new file mode 100644 index 000000000..70e1cf627 --- /dev/null +++ b/src/lsp/ProjectManager.spec.ts @@ -0,0 +1,85 @@ +import { expect } from 'chai'; +import { ProjectManager } from './ProjectManager'; +import { tempDir, rootDir } from '../testHelpers.spec'; +import * as fsExtra from 'fs-extra'; +import { standardizePath as s } from '../util'; + +describe.only('ProjectManager', () => { + let manager: ProjectManager; + + beforeEach(() => { + manager = new ProjectManager(); + fsExtra.emptyDirSync(tempDir); + }); + + afterEach(() => { + fsExtra.emptyDirSync(tempDir); + }); + + describe('syncProjects', () => { + it('does not crash on zero projects', async () => { + await manager.syncProjects([]); + }); + + it('finds bsconfig in a folder', async () => { + fsExtra.outputFileSync(`${rootDir}/bsconfig.json`, ''); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect(manager.projects[0].projectPath).to.eql(s`${rootDir}`); + }); + + it('finds bsconfig at root and also in subfolder', async () => { + fsExtra.outputFileSync(`${rootDir}/bsconfig.json`, ''); + fsExtra.outputFileSync(`${rootDir}/subdir/bsconfig.json`, ''); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect( + manager.projects.map(x => x.projectPath) + ).to.eql([ + s`${rootDir}`, + s`${rootDir}/subdir` + ]); + }); + + it('skips excluded bsconfig bsconfig in a folder', async () => { + fsExtra.outputFileSync(`${rootDir}/bsconfig.json`, ''); + fsExtra.outputFileSync(`${rootDir}/subdir/bsconfig.json`, ''); + await manager.syncProjects([{ + workspaceFolder: rootDir, + excludePatterns: ['subdir/**/*'] + }]); + expect( + manager.projects.map(x => x.projectPath) + ).to.eql([ + s`${rootDir}` + ]); + }); + + it('uses rootDir when manifest found but no brightscript file', async () => { + fsExtra.outputFileSync(`${rootDir}/subdir/manifest`, ''); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect( + manager.projects.map(x => x.projectPath) + ).to.eql([ + s`${rootDir}` + ]); + }); + + it('uses subdir when manifest and brightscript file found', async () => { + fsExtra.outputFileSync(`${rootDir}/subdir/manifest`, ''); + fsExtra.outputFileSync(`${rootDir}/subdir/source/main.brs`, ''); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect( + manager.projects.map(x => x.projectPath) + ).to.eql([ + s`${rootDir}/subdir` + ]); + }); + }); +}); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts new file mode 100644 index 000000000..7c73ba639 --- /dev/null +++ b/src/lsp/ProjectManager.ts @@ -0,0 +1,321 @@ +import type { Project } from './Project'; +import { standardizePath as s, util } from '../util'; +import { rokuDeploy } from 'roku-deploy'; +import * as path from 'path'; +import { ProgramBuilder } from '../ProgramBuilder'; +import * as EventEmitter from 'eventemitter3'; +import type { CompilerPlugin, FileResolver } from '../interfaces'; +import { Deferred } from '../deferred'; +import { DiagnosticMessages } from '../DiagnosticMessages'; + +/** + * Manages all brighterscript projects for the language server + */ +export class ProjectManager { + + /** + * Collection of all projects + */ + public projects: Project[] = []; + + /** + * A unique project counter to help distinguish log entries in lsp mode + */ + private projectCounter = 0; + + private emitter = new EventEmitter(); + + public on(eventName: 'flush-diagnostics', handler: (data: { project: Project }) => void); + public on(eventName: string, handler: (payload: any) => void) { + this.emitter.on(eventName, handler); + return () => { + this.emitter.removeListener(eventName, handler); + }; + } + + private emit(eventName: 'critical-failure', data: { project: Project; message: string }); + private emit(eventName: 'flush-diagnostics', data: { project: Project }); + private async emit(eventName: string, data?) { + //emit these events on next tick, otherwise they will be processed immediately which could cause issues + await util.sleep(0); + this.emitter.emit(eventName, data); + } + + private fileResolvers = [] as FileResolver[]; + + public addFileResolver(fileResolver: FileResolver) { + this.fileResolvers.push(fileResolver); + } + + /** + * Given a list of all desired projects, create any missing projects and destroy and projects that are no longer available + * Treat workspaces that don't have a bsconfig.json as a project. + * Handle situations where bsconfig.json files were added or removed (to elevate/lower workspaceFolder projects accordingly) + * Leave existing projects alone if they are not affected by these changes + * @param workspaceConfigs an array of workspaces + */ + public async syncProjects(workspaceConfigs: WorkspaceConfig[]) { + //build a list of unique projects + let projectConfigs = (await Promise.all( + workspaceConfigs.map(async workspaceConfig => { + const projectPaths = await this.getProjectPaths(workspaceConfig); + return projectPaths.map(projectPath => ({ + projectPath: s`${projectPath}`, + workspaceFolder: s`${workspaceConfig}`, + excludePatterns: workspaceConfig.excludePatterns + })); + }) + )).flat(1); + + //delete projects not represented in the list + for (const project of this.projects) { + //we can't find this existing project in our new list, so scrap it + if (!projectConfigs.find(x => x.projectPath === project.projectPath)) { + this.removeProject(project); + } + } + + // skip projects we already have (they're already loaded...no need to reload them) + projectConfigs = projectConfigs.filter(x => { + return !this.hasProject(x.projectPath); + }); + + //dedupe by project path + projectConfigs = [ + ...projectConfigs.reduce( + (acc, x) => acc.set(x.projectPath, x), + new Map() + ).values() + ]; + + //create missing projects + await Promise.all( + projectConfigs.map(config => this.createProject(config)) + ); + } + + /** + * Scan a given workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. + * If none are found, then the workspaceFolder itself is treated as a project + */ + private async getProjectPaths(workspaceConfig: WorkspaceConfig) { + //get the list of exclude patterns, and negate them (so they actually work like excludes) + const excludePatterns = (workspaceConfig.excludePatterns ?? []).map(x => s`!${x}`); + const files = await rokuDeploy.getFilePaths([ + '**/bsconfig.json', + //exclude all files found in `files.exclude` + ...excludePatterns + ], workspaceConfig.workspaceFolder); + + //if we found at least one bsconfig.json, then ALL projects must have a bsconfig.json. + if (files.length > 0) { + return files.map(file => s`${path.dirname(file.src)}`); + } + + //look for roku project folders + const rokuLikeDirs = (await Promise.all( + //find all folders containing a `manifest` file + (await rokuDeploy.getFilePaths([ + '**/manifest', + ...excludePatterns + + //is there at least one .bs|.brs file under the `/source` folder? + ], workspaceConfig.workspaceFolder)).map(async manifestEntry => { + const manifestDir = path.dirname(manifestEntry.src); + const files = await rokuDeploy.getFilePaths([ + 'source/**/*.{brs,bs}', + ...excludePatterns + ], manifestDir); + if (files.length > 0) { + return manifestDir; + } + }) + //throw out nulls + )).filter(x => !!x); + if (rokuLikeDirs.length > 0) { + return rokuLikeDirs; + } + + //treat the workspace folder as a brightscript project itself + return [workspaceConfig.workspaceFolder]; + } + + /** + * Returns true if we have this project, or false if we don't + * @param projectPath path to the project + * @returns true if the project exists, or false if it doesn't + */ + private hasProject(projectPath: string) { + return !!this.getProject(projectPath); + } + + /** + * Get a project with the specified path + * @param param path to the project or an obj that has `projectPath` prop + * @returns a project, or undefined if no project was found + */ + private getProject(param: string | { projectPath: string }) { + const projectPath = (typeof param === 'string') ? param : param.projectPath; + return this.projects.find(x => x.projectPath === s`${projectPath}`); + } + + /** + * Remove a project from the language server + */ + private removeProject(project: Project) { + const idx = this.projects.findIndex(x => x.projectPath === project?.projectPath); + if (idx > -1) { + this.projects.splice(idx, 1); + } + project?.builder?.dispose(); + } + + private async createProject(config: ProjectConfig) { + config.workspaceFolder ??= config.projectPath; + + //skip this project if we already have it + if (this.hasProject(config.projectPath)) { + return; + } + + let builder = new ProgramBuilder(); + + config.projectNumber ??= this.projectCounter++; + + builder.logger.prefix = `[prj${config.projectNumber}]`; + builder.logger.log(`Created project #${config.projectNumber} for: "${config.projectPath}"`); + + //flush diagnostics every time the program finishes validating + builder.plugins.add({ + name: 'bsc-language-server', + afterProgramValidate: () => { + this.emit('flush-diagnostics', { project: this.getProject(config) }); + } + } as CompilerPlugin); + + //prevent clearing the console on run...this isn't the CLI so we want to keep a full log of everything + builder.allowConsoleClearing = false; + + //register any external file resolvers + builder.addFileResolver(...this.fileResolvers); + + let configFilePath = await this.getBsconfigPath(config); + + let cwd = config.projectPath; + + //if the config file exists, use it and its folder as cwd + if (configFilePath && await util.pathExists(configFilePath)) { + cwd = path.dirname(configFilePath); + } else { + //config file doesn't exist...let `brighterscript` resolve the default way + configFilePath = undefined; + } + + const firstRunDeferred = new Deferred(); + + let newProject: Project = { + projectNumber: config.projectNumber, + builder: builder, + firstRunPromise: firstRunDeferred.promise, + projectPath: config.projectPath, + workspaceFolder: config.workspaceFolder, + isFirstRunComplete: false, + isFirstRunSuccessful: false, + configFilePath: configFilePath, + isStandaloneFileProject: false + }; + + this.projects.push(newProject); + + try { + await builder.run({ + cwd: cwd, + project: configFilePath, + watch: false, + createPackage: false, + deploy: false, + copyToStaging: false, + showDiagnosticsInConsole: false + }); + newProject.isFirstRunComplete = true; + newProject.isFirstRunSuccessful = true; + firstRunDeferred.resolve(); + } catch (e) { + builder.logger.error(e); + firstRunDeferred.reject(e); + newProject.isFirstRunComplete = true; + newProject.isFirstRunSuccessful = false; + } + //if we found a deprecated brsconfig.json, add a diagnostic warning the user + if (configFilePath && path.basename(configFilePath) === 'brsconfig.json') { + builder.addDiagnostic(configFilePath, { + ...DiagnosticMessages.brsConfigJsonIsDeprecated(), + range: util.createRange(0, 0, 0, 0) + }); + this.emit('flush-diagnostics', { project: newProject }); + } + } + + private async getBsconfigPath(config: ProjectConfig) { + let configFilePath: string; + //if there's a setting, we need to find the file or show error if it can't be found + if (config?.bsconfigPath) { + configFilePath = path.resolve(config.projectPath, config?.bsconfigPath); + if (await util.pathExists(configFilePath)) { + return configFilePath; + } else { + this.emit('critical-failure', { + message: `Cannot find config file specified in user / workspace settings at '${configFilePath}'`, + project: this.getProject(config) + }); + } + } + + //default to config file path found in the root of the workspace + configFilePath = path.resolve(config.projectPath, 'bsconfig.json'); + if (await util.pathExists(configFilePath)) { + return configFilePath; + } + + //look for the deprecated `brsconfig.json` file + configFilePath = path.resolve(config.projectPath, 'brsconfig.json'); + if (await util.pathExists(configFilePath)) { + return configFilePath; + } + + //no config file could be found + return undefined; + } +} + +interface WorkspaceConfig { + workspaceFolder: string; + excludePatterns?: string[]; + /** + * Path to a bsconfig that should be used instead of the auto-discovery algorithm. If this is present, no bsconfig discovery should be used. and an error should be emitted if this file is missing + */ + bsconfigPath?: string; +} + +interface ProjectConfig { + /** + * Path to the project + */ + projectPath: string; + /** + * Path to the workspace in which all project files reside or are referenced by + */ + workspaceFolder: string; + /** + * A list of glob patterns used to _exclude_ files from various bsconfig searches + */ + excludePatterns: string[]; + /** + * An optional project number to assign to the project within the context of a language server. reloaded projects should keep the same number if possible + */ + projectNumber?: number; + /** + * Path to a bsconfig that should be used instead of the auto-discovery algorithm. If this is present, no bsconfig discovery should be used. and an error should be emitted if this file is missing + */ + bsconfigPath?: string; +} From eddf081394b8a9254d08135092833238655bd389 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Thu, 21 Dec 2023 21:58:41 -0500 Subject: [PATCH 002/119] 100% coverage on current ProjectManager --- src/ProgramBuilder.ts | 4 +- src/lsp/ProjectManager.spec.ts | 185 ++++++++++++++++++++++++++++++++- src/lsp/ProjectManager.ts | 18 +++- 3 files changed, 198 insertions(+), 9 deletions(-) diff --git a/src/ProgramBuilder.ts b/src/ProgramBuilder.ts index 5cdf5efcc..0862edc60 100644 --- a/src/ProgramBuilder.ts +++ b/src/ProgramBuilder.ts @@ -43,7 +43,7 @@ export class ProgramBuilder { /** * Add file resolvers that will be able to provide file contents before loading from the file system - * @param fileResolvers + * @param fileResolvers a list of file resolvers */ public addFileResolver(...fileResolvers: FileResolver[]) { this.fileResolvers.push(...fileResolvers); @@ -75,7 +75,7 @@ export class ProgramBuilder { let file: BscFile = this.program.getFile(srcPath); if (!file) { file = { - pkgPath: this.program.getPkgPath(srcPath), + pkgPath: path.basename(srcPath), pathAbsolute: srcPath, //keep this for backwards-compatibility. TODO remove in v1 srcPath: srcPath, getDiagnostics: () => { diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index 70e1cf627..b298d304c 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -1,8 +1,13 @@ import { expect } from 'chai'; import { ProjectManager } from './ProjectManager'; -import { tempDir, rootDir } from '../testHelpers.spec'; +import { tempDir, rootDir, expectDiagnostics } from '../testHelpers.spec'; import * as fsExtra from 'fs-extra'; import { standardizePath as s } from '../util'; +import { Deferred } from '../deferred'; +import { createSandbox } from 'sinon'; +import { DiagnosticMessages } from '../DiagnosticMessages'; +import { ProgramBuilder } from '..'; +const sinon = createSandbox(); describe.only('ProjectManager', () => { let manager: ProjectManager; @@ -16,6 +21,37 @@ describe.only('ProjectManager', () => { fsExtra.emptyDirSync(tempDir); }); + describe('addFileResolver', () => { + it('runs added resolvers', async () => { + const mock = sinon.mock(); + manager.addFileResolver(mock); + fsExtra.outputFileSync(`${rootDir}/source/main.brs`, ''); + + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect(mock.called).to.be.true; + }); + }); + + describe('events', () => { + it('emits flush-diagnostics after validate finishes', async () => { + const deferred = new Deferred(); + const disable = manager.on('flush-diagnostics', () => { + deferred.resolve(true); + }); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect( + await deferred.promise + ).to.eql(true); + + //disable future events + disable(); + }); + }); + describe('syncProjects', () => { it('does not crash on zero projects', async () => { await manager.syncProjects([]); @@ -36,7 +72,7 @@ describe.only('ProjectManager', () => { workspaceFolder: rootDir }]); expect( - manager.projects.map(x => x.projectPath) + manager.projects.map(x => x.projectPath).sort() ).to.eql([ s`${rootDir}`, s`${rootDir}/subdir` @@ -81,5 +117,150 @@ describe.only('ProjectManager', () => { s`${rootDir}/subdir` ]); }); + + it('removes stale projects', async () => { + fsExtra.outputFileSync(`${rootDir}/subdir1/bsconfig.json`, ''); + fsExtra.outputFileSync(`${rootDir}/subdir2/bsconfig.json`, ''); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect( + manager.projects.map(x => x.projectPath).sort() + ).to.eql([ + s`${rootDir}/subdir1`, + s`${rootDir}/subdir2` + ]); + fsExtra.removeSync(`${rootDir}/subdir1/bsconfig.json`); + + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect( + manager.projects.map(x => x.projectPath).sort() + ).to.eql([ + s`${rootDir}/subdir2` + ]); + }); + + it('keeps existing projects on subsequent sync calls', async () => { + fsExtra.outputFileSync(`${rootDir}/subdir1/bsconfig.json`, ''); + fsExtra.outputFileSync(`${rootDir}/subdir2/bsconfig.json`, ''); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect( + manager.projects.map(x => x.projectPath).sort() + ).to.eql([ + s`${rootDir}/subdir1`, + s`${rootDir}/subdir2` + ]); + + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect( + manager.projects.map(x => x.projectPath).sort() + ).to.eql([ + s`${rootDir}/subdir1`, + s`${rootDir}/subdir2` + ]); + }); + }); + + describe('createProject', () => { + it('skips creating project if we already have it', async () => { + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + + await manager['createProject']({ + projectPath: rootDir + } as any); + expect(manager.projects).to.be.length(1); + }); + + it('uses given projectNumber', async () => { + await manager['createProject']({ + projectPath: rootDir, + workspaceFolder: rootDir, + projectNumber: 3 + }); + expect(manager.projects[0].projectNumber).to.eql(3); + }); + + it('warns about deprecated brsconfig.json', async () => { + fsExtra.outputFileSync(`${rootDir}/subdir1/brsconfig.json`, ''); + const project = await manager['createProject']({ + projectPath: rootDir, + workspaceFolder: rootDir, + bsconfigPath: 'subdir1/brsconfig.json' + }); + expectDiagnostics(project.builder, [ + DiagnosticMessages.brsConfigJsonIsDeprecated() + ]); + }); + + it('properly tracks a failed run', async () => { + //force a total crash + sinon.stub(ProgramBuilder.prototype, 'run').returns( + Promise.reject(new Error('Critical failure')) + ); + const project = await manager['createProject']({ + projectPath: rootDir, + workspaceFolder: rootDir, + bsconfigPath: 'subdir1/brsconfig.json' + }); + expect(project.isFirstRunComplete).to.be.true; + expect(project.isFirstRunSuccessful).to.be.false; + }); + }); + + describe('getBsconfigPath', () => { + it('emits critical failure for missing file', async () => { + const deferred = new Deferred(); + manager.on('critical-failure', (event) => { + deferred.resolve(event.message); + }); + await manager['getBsconfigPath']({ + projectPath: rootDir, + workspaceFolder: rootDir, + bsconfigPath: s`${rootDir}/bsconfig.json`, + }); + expect( + (await deferred.promise).startsWith('Cannot find config file') + ).to.be.true; + }); + + it('finds brsconfig.json', async () => { + fsExtra.outputFileSync(`${rootDir}/brsconfig.json`, ''); + expect( + await manager['getBsconfigPath']({ + projectPath: rootDir, + workspaceFolder: rootDir + }) + ).to.eql(s`${rootDir}/brsconfig.json`); + }); + + + it('does not crash on undefined', async () => { + await manager['getBsconfigPath'](undefined) + }); + }); + + describe('removeProject', () => { + it('handles undefined', async () => { + manager['removeProject'](undefined); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + manager['removeProject'](undefined); + }); + + it('does not crash when removing project that is not there', () => { + manager['removeProject']({ + projectPath: rootDir, + dispose: () => { } + } as any); + }); }); }); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 7c73ba639..c5226f63b 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -25,6 +25,7 @@ export class ProjectManager { private emitter = new EventEmitter(); + public on(eventName: 'critical-failure', handler: (data: { project: Project; message: string }) => void); public on(eventName: 'flush-diagnostics', handler: (data: { project: Project }) => void); public on(eventName: string, handler: (payload: any) => void) { this.emitter.on(eventName, handler); @@ -175,7 +176,7 @@ export class ProjectManager { //skip this project if we already have it if (this.hasProject(config.projectPath)) { - return; + return this.getProject(config.projectPath); } let builder = new ProgramBuilder(); @@ -254,13 +255,15 @@ export class ProjectManager { }); this.emit('flush-diagnostics', { project: newProject }); } + return newProject; } private async getBsconfigPath(config: ProjectConfig) { + let configFilePath: string; //if there's a setting, we need to find the file or show error if it can't be found if (config?.bsconfigPath) { - configFilePath = path.resolve(config.projectPath, config?.bsconfigPath); + configFilePath = path.resolve(config.projectPath, config.bsconfigPath); if (await util.pathExists(configFilePath)) { return configFilePath; } else { @@ -271,14 +274,19 @@ export class ProjectManager { } } + //the rest of these require a projectPath, so return early if we don't have one + if (!config?.projectPath) { + return undefined; + } + //default to config file path found in the root of the workspace - configFilePath = path.resolve(config.projectPath, 'bsconfig.json'); + configFilePath = s`${config.projectPath}/bsconfig.json`; if (await util.pathExists(configFilePath)) { return configFilePath; } //look for the deprecated `brsconfig.json` file - configFilePath = path.resolve(config.projectPath, 'brsconfig.json'); + configFilePath = s`${config.projectPath}/brsconfig.json`; if (await util.pathExists(configFilePath)) { return configFilePath; } @@ -309,7 +317,7 @@ interface ProjectConfig { /** * A list of glob patterns used to _exclude_ files from various bsconfig searches */ - excludePatterns: string[]; + excludePatterns?: string[]; /** * An optional project number to assign to the project within the context of a language server. reloaded projects should keep the same number if possible */ From 2352f365c4908669861105c11b0c6ca6388ec8aa Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Fri, 22 Dec 2023 16:23:56 -0500 Subject: [PATCH 003/119] Significant refactors. Added worker project support (it's broke!) --- src/LanguageServer.ts | 86 ++++----- src/interfaces.ts | 4 +- src/lsp/LspProject.ts | 56 ++++++ src/lsp/Project.spec.ts | 136 ++++++++++++++ src/lsp/Project.ts | 170 +++++++++++++++-- src/lsp/ProjectManager.spec.ts | 11 +- src/lsp/ProjectManager.ts | 191 ++++++-------------- src/lsp/worker/MessageHandler.spec.ts | 40 ++++ src/lsp/worker/MessageHandler.ts | 176 ++++++++++++++++++ src/lsp/worker/WorkerThreadProject.spec.ts | 51 ++++++ src/lsp/worker/WorkerThreadProject.ts | 112 ++++++++++++ src/lsp/worker/WorkerThreadProjectRunner.ts | 32 ++++ src/testHelpers.spec.ts | 12 ++ src/util.ts | 20 +- 14 files changed, 899 insertions(+), 198 deletions(-) create mode 100644 src/lsp/LspProject.ts create mode 100644 src/lsp/Project.spec.ts create mode 100644 src/lsp/worker/MessageHandler.spec.ts create mode 100644 src/lsp/worker/MessageHandler.ts create mode 100644 src/lsp/worker/WorkerThreadProject.spec.ts create mode 100644 src/lsp/worker/WorkerThreadProject.ts create mode 100644 src/lsp/worker/WorkerThreadProjectRunner.ts diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index f721d9fa0..be2e15936 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -46,6 +46,7 @@ import { isBrsFile } from './astUtils/reflection'; import { encodeSemanticTokens, semanticTokensLegend } from './SemanticTokenUtils'; import type { BusyStatus } from './BusyStatusTracker'; import { BusyStatusTracker } from './BusyStatusTracker'; +import type { WorkspaceConfig } from './lsp/ProjectManager'; import { ProjectManager } from './lsp/ProjectManager'; export class LanguageServer { @@ -256,6 +257,44 @@ export class LanguageServer { }; } + /** + * Called when the client has finished initializing + */ + @AddStackToErrorMessage + @TrackBusyStatus + private async onInitialized() { + let projectCreatedDeferred = new Deferred(); + this.initialProjectsCreated = projectCreatedDeferred.promise; + + try { + if (this.hasConfigurationCapability) { + // Register for all configuration changes. + await this.connection.client.register( + DidChangeConfigurationNotification.type, + undefined + ); + } + + await this.syncProjects(); + + if (this.clientHasWorkspaceFolderCapability) { + this.connection.workspace.onDidChangeWorkspaceFolders(async (evt) => { + await this.syncProjects(); + }); + } + await this.waitAllProjectFirstRuns(false); + projectCreatedDeferred.resolve(); + } catch (e: any) { + this.sendCriticalFailure( + `Critical failure during BrighterScript language server startup. + Please file a github issue and include the contents of the 'BrighterScript Language Server' output channel. + + Error message: ${e.message}` + ); + throw e; + } + } + private initialProjectsCreated: Promise; /** @@ -293,15 +332,18 @@ export class LanguageServer { let workspaces = await Promise.all( (await this.connection.workspace.getWorkspaceFolders() ?? []).map(async (x) => { const workspaceFolder = util.uriToPath(x.uri); + const config = await this.getClientConfiguration(x.uri, 'brightscript'); return { workspaceFolder: workspaceFolder, excludePatterns: await this.getWorkspaceExcludeGlobs(workspaceFolder), - bsconfigPath: (await this.getClientConfiguration(x.uri, 'brightscript'))?.configFile - }; + bsconfigPath: config.configFile, + //TODO we need to solidify the actual name of this flag in user/workspace settings + threadingEnabled: config.threadingEnabled + + } as WorkspaceConfig; }) ); - await this.projectManager.syncProjects(workspaces); //flush diagnostics @@ -332,44 +374,6 @@ export class LanguageServer { return config as T; } - /** - * Called when the client has finished initializing - */ - @AddStackToErrorMessage - @TrackBusyStatus - private async onInitialized() { - let projectCreatedDeferred = new Deferred(); - this.initialProjectsCreated = projectCreatedDeferred.promise; - - try { - if (this.hasConfigurationCapability) { - // Register for all configuration changes. - await this.connection.client.register( - DidChangeConfigurationNotification.type, - undefined - ); - } - - await this.syncProjects(); - - if (this.clientHasWorkspaceFolderCapability) { - this.connection.workspace.onDidChangeWorkspaceFolders(async (evt) => { - await this.syncProjects(); - }); - } - await this.waitAllProjectFirstRuns(false); - projectCreatedDeferred.resolve(); - } catch (e: any) { - this.sendCriticalFailure( - `Critical failure during BrighterScript language server startup. - Please file a github issue and include the contents of the 'BrighterScript Language Server' output channel. - - Error message: ${e.message}` - ); - throw e; - } - } - /** * Send a critical failure notification to the client, which should show a notification of some kind */ diff --git a/src/interfaces.ts b/src/interfaces.ts index 9ac1a2b7b..248d9e128 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -1,4 +1,4 @@ -import type { Range, Diagnostic, CodeAction, SemanticTokenTypes, SemanticTokenModifiers, Position, CompletionItem } from 'vscode-languageserver'; +import type { Range, Diagnostic, CodeAction, SemanticTokenTypes, SemanticTokenModifiers, Position, CompletionItem, Disposable } from 'vscode-languageserver'; import type { Scope } from './Scope'; import type { BrsFile } from './files/BrsFile'; import type { XmlFile } from './files/XmlFile'; @@ -411,3 +411,5 @@ export interface FileLink { item: T; file: BrsFile; } + +export type DisposableLike = Disposable | (() => any); diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts new file mode 100644 index 000000000..ac5f71be8 --- /dev/null +++ b/src/lsp/LspProject.ts @@ -0,0 +1,56 @@ +import type { Diagnostic } from 'vscode-languageserver-types'; + +/** + * Defines the contract between the ProjectManager and the main or worker thread Project classes + */ +export interface LspProject { + + /** + * The path to where the project resides + */ + projectPath: string; + + /** + * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging + */ + projectNumber: number; + + /** + * Initialize and start running the project. This will scan for all files, and build a full project in memory, then validate the project + * @param options + */ + activate(options: ActivateOptions): Promise; + + /** + * Get the list of all diagnostics from this project + */ + getDiagnostics(): Promise; + + /** + * Release all resources so this file can be safely garbage collected + */ + dispose(): void; +} + +export interface ActivateOptions { + /** + * The path to where the project resides + */ + projectPath: string; + /** + * The path to the workspace where this project resides. A workspace can have multiple projects (by adding a bsconfig.json to each folder). + */ + workspaceFolder?: string; + /** + * Path to a bsconfig.json file that shall be used for this project + */ + configFilePath?: string; + /** + * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging + */ + projectNumber?: number; +} + +export interface LspDiagnostic extends Diagnostic { + uri: string; +} diff --git a/src/lsp/Project.spec.ts b/src/lsp/Project.spec.ts new file mode 100644 index 000000000..f4a735844 --- /dev/null +++ b/src/lsp/Project.spec.ts @@ -0,0 +1,136 @@ +import { expect } from 'chai'; +import { tempDir, rootDir, expectDiagnosticsAsync } from '../testHelpers.spec'; +import * as fsExtra from 'fs-extra'; +import { standardizePath as s } from '../util'; +import { Deferred } from '../deferred'; +import { createSandbox } from 'sinon'; +import { DiagnosticMessages } from '../DiagnosticMessages'; +import { ProgramBuilder } from '..'; +import { Project } from './Project'; +const sinon = createSandbox(); + +describe('ProjectManager', () => { + let project: Project; + + beforeEach(() => { + project = new Project(); + fsExtra.emptyDirSync(tempDir); + }); + + afterEach(() => { + fsExtra.emptyDirSync(tempDir); + project.dispose(); + }); + + describe('activate', () => { + it('finds bsconfig.json at root', async () => { + fsExtra.outputFileSync(`${rootDir}/bsconfig.json`, ''); + await project.activate({ + projectPath: rootDir + }); + expect(project.configFilePath).to.eql(s`${rootDir}/bsconfig.json`); + }); + + it('shows diagnostics after running', async () => { + fsExtra.outputFileSync(`${rootDir}/source/main.brs`, ` + sub main() + print varNotThere + end sub + `); + + await project.activate({ + projectPath: rootDir + }); + + await expectDiagnosticsAsync(project, [ + DiagnosticMessages.cannotFindName('varNotThere').message + ]); + }); + }); + + describe('createProject', () => { + it('uses given projectNumber', async () => { + await manager['createProject']({ + projectPath: rootDir, + workspaceFolder: rootDir, + projectNumber: 3 + }); + expect(manager.projects[0].projectNumber).to.eql(3); + }); + + it('warns about deprecated brsconfig.json', async () => { + fsExtra.outputFileSync(`${rootDir}/subdir1/brsconfig.json`, ''); + const project = await manager['createProject']({ + projectPath: rootDir, + workspaceFolder: rootDir, + bsconfigPath: 'subdir1/brsconfig.json' + }); + await expectDiagnosticsAsync(project, [ + DiagnosticMessages.brsConfigJsonIsDeprecated() + ]); + }); + + it('properly tracks a failed run', async () => { + //force a total crash + sinon.stub(ProgramBuilder.prototype, 'run').returns( + Promise.reject(new Error('Critical failure')) + ); + const project = await manager['createProject']({ + projectPath: rootDir, + workspaceFolder: rootDir, + bsconfigPath: 'subdir1/brsconfig.json' + }); + expect(project.isFirstRunComplete).to.be.true; + expect(project.isFirstRunSuccessful).to.be.false; + }); + }); + + describe('getBsconfigPath', () => { + it('emits critical failure for missing file', async () => { + const deferred = new Deferred(); + manager.on('critical-failure', (event) => { + deferred.resolve(event.message); + }); + await manager['getBsconfigPath']({ + projectPath: rootDir, + workspaceFolder: rootDir, + bsconfigPath: s`${rootDir}/bsconfig.json` + }); + expect( + (await deferred.promise).startsWith('Cannot find config file') + ).to.be.true; + }); + + it('finds brsconfig.json', async () => { + fsExtra.outputFileSync(`${rootDir}/brsconfig.json`, ''); + expect( + await manager['getBsconfigPath']({ + projectPath: rootDir, + workspaceFolder: rootDir + }) + ).to.eql(s`${rootDir}/brsconfig.json`); + }); + + + it('does not crash on undefined', async () => { + await manager['getBsconfigPath'](undefined); + }); + }); + + describe('removeProject', () => { + it('handles undefined', async () => { + manager['removeProject'](undefined); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + manager['removeProject'](undefined); + }); + + it('does not crash when removing project that is not there', () => { + manager['removeProject']({ + projectPath: rootDir, + dispose: () => { } + } as any); + }); + }); +}); diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 42a792b9a..720bf5012 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -1,22 +1,168 @@ -import type { ProgramBuilder } from '../ProgramBuilder'; +import { ProgramBuilder } from '../ProgramBuilder'; +import * as EventEmitter from 'eventemitter3'; +import util, { standardizePath as s } from '../util'; +import * as path from 'path'; +import type { ActivateOptions, LspProject } from './LspProject'; +import type { CompilerPlugin } from '../interfaces'; +import { DiagnosticMessages } from '../DiagnosticMessages'; +import { URI } from 'vscode-uri'; + +export class Project implements LspProject { + + public async activate(options: ActivateOptions) { + this.projectPath = options.projectPath; + this.workspaceFolder = options.workspaceFolder; + this.projectNumber = options.projectNumber; + this.configFilePath = await this.getConfigFilePath(options); + + this.builder = new ProgramBuilder(); + this.builder.logger.prefix = `[prj${this.projectNumber}]`; + this.builder.logger.log(`Created project #${this.projectNumber} for: "${this.projectPath}"`); + + let cwd; + //if the config file exists, use it and its folder as cwd + if (this.configFilePath && await util.pathExists(this.configFilePath)) { + cwd = path.dirname(this.configFilePath); + } else { + cwd = this.projectPath; + //config file doesn't exist...let `brighterscript` resolve the default way + this.configFilePath = undefined; + } + + //flush diagnostics every time the program finishes validating + this.builder.plugins.add({ + name: 'bsc-language-server', + afterProgramValidate: () => { + this.emit('flush-diagnostics', { project: this }); + } + } as CompilerPlugin); + + //register any external file resolvers + //TODO handle in-memory file stuff + // builder.addFileResolver(...this.fileResolvers); + + try { + await this.builder.run({ + cwd: cwd, + project: this.configFilePath, + watch: false, + createPackage: false, + deploy: false, + copyToStaging: false, + showDiagnosticsInConsole: false + }); + } catch (e) { + this.builder.logger.error(e); + } + + //if we found a deprecated brsconfig.json, add a diagnostic warning the user + if (this.configFilePath && path.basename(this.configFilePath) === 'brsconfig.json') { + this.builder.addDiagnostic(this.configFilePath, { + ...DiagnosticMessages.brsConfigJsonIsDeprecated(), + range: util.createRange(0, 0, 0, 0) + }); + this.emit('flush-diagnostics', { project: this }); + } + } + + public getDiagnostics() { + return Promise.resolve( + this.builder.getDiagnostics().map(x => { + const uri = URI.file(x.file.srcPath).toString(); + return { + ...util.toDiagnostic(x, uri), + uri: uri + }; + }) + ); + } + + public dispose() { + } -export interface Project { /** - * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging + * Manages the BrighterScript program. The main interface into the compiler/validator */ - projectNumber: number; - firstRunPromise: Promise; - builder: ProgramBuilder; + private builder = new ProgramBuilder(); + /** * The path to where the project resides */ - projectPath: string; + public projectPath: string; + + /** + * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging + */ + public projectNumber: number; + /** * The path to the workspace where this project resides. A workspace can have multiple projects (by adding a bsconfig.json to each folder). + * Defaults to `.projectPath` if not set + */ + public workspaceFolder: string; + + /** + * Path to a bsconfig.json file that will be used for this project */ - workspaceFolder: string; - isFirstRunComplete: boolean; - isFirstRunSuccessful: boolean; - configFilePath?: string; - isStandaloneFileProject: boolean; + public configFilePath?: string; + + + /** + * Find the path to the bsconfig.json file for this project + * @param config options that help us find the bsconfig.json + * @returns path to bsconfig.json, or undefined if unable to find it + */ + private async getConfigFilePath(config: { configFilePath?: string; projectPath: string }) { + let configFilePath: string; + //if there's a setting, we need to find the file or show error if it can't be found + if (config?.configFilePath) { + configFilePath = path.resolve(config.projectPath, config.configFilePath); + if (await util.pathExists(configFilePath)) { + return configFilePath; + } else { + this.emit('critical-failure', { + message: `Cannot find config file specified in user or workspace settings at '${configFilePath}'`, + project: this + }); + } + } + + //the rest of these require a projectPath, so return early if we don't have one + if (!config?.projectPath) { + return undefined; + } + + //default to config file path found in the root of the workspace + configFilePath = s`${config.projectPath}/bsconfig.json`; + if (await util.pathExists(configFilePath)) { + return configFilePath; + } + + //look for the deprecated `brsconfig.json` file + configFilePath = s`${config.projectPath}/brsconfig.json`; + if (await util.pathExists(configFilePath)) { + return configFilePath; + } + + //no config file could be found + return undefined; + } + + public on(eventName: 'critical-failure', handler: (data: { project: Project; message: string }) => void); + public on(eventName: 'flush-diagnostics', handler: (data: { project: Project }) => void); + public on(eventName: string, handler: (payload: any) => void) { + this.emitter.on(eventName, handler); + return () => { + this.emitter.removeListener(eventName, handler); + }; + } + + private emit(eventName: 'critical-failure', data: { project: Project; message: string }); + private emit(eventName: 'flush-diagnostics', data: { project: Project }); + private async emit(eventName: string, data?) { + //emit these events on next tick, otherwise they will be processed immediately which could cause issues + await util.sleep(0); + this.emitter.emit(eventName, data); + } + private emitter = new EventEmitter(); } diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index b298d304c..fd393c3ce 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -1,6 +1,6 @@ import { expect } from 'chai'; import { ProjectManager } from './ProjectManager'; -import { tempDir, rootDir, expectDiagnostics } from '../testHelpers.spec'; +import { tempDir, rootDir, expectDiagnostics, expectDiagnosticsAsync } from '../testHelpers.spec'; import * as fsExtra from 'fs-extra'; import { standardizePath as s } from '../util'; import { Deferred } from '../deferred'; @@ -9,7 +9,7 @@ import { DiagnosticMessages } from '../DiagnosticMessages'; import { ProgramBuilder } from '..'; const sinon = createSandbox(); -describe.only('ProjectManager', () => { +describe('ProjectManager', () => { let manager: ProjectManager; beforeEach(() => { @@ -195,7 +195,7 @@ describe.only('ProjectManager', () => { workspaceFolder: rootDir, bsconfigPath: 'subdir1/brsconfig.json' }); - expectDiagnostics(project.builder, [ + await expectDiagnosticsAsync(project, [ DiagnosticMessages.brsConfigJsonIsDeprecated() ]); }); @@ -224,7 +224,7 @@ describe.only('ProjectManager', () => { await manager['getBsconfigPath']({ projectPath: rootDir, workspaceFolder: rootDir, - bsconfigPath: s`${rootDir}/bsconfig.json`, + bsconfigPath: s`${rootDir}/bsconfig.json` }); expect( (await deferred.promise).startsWith('Cannot find config file') @@ -241,9 +241,8 @@ describe.only('ProjectManager', () => { ).to.eql(s`${rootDir}/brsconfig.json`); }); - it('does not crash on undefined', async () => { - await manager['getBsconfigPath'](undefined) + await manager['getBsconfigPath'](undefined); }); }); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index c5226f63b..0deff9d84 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -1,12 +1,18 @@ -import type { Project } from './Project'; import { standardizePath as s, util } from '../util'; import { rokuDeploy } from 'roku-deploy'; import * as path from 'path'; -import { ProgramBuilder } from '../ProgramBuilder'; import * as EventEmitter from 'eventemitter3'; -import type { CompilerPlugin, FileResolver } from '../interfaces'; -import { Deferred } from '../deferred'; -import { DiagnosticMessages } from '../DiagnosticMessages'; +import type { FileResolver } from '../interfaces'; +import type { LspProject } from './LspProject'; +import { Project } from './Project'; +import { WorkerThreadProject } from './worker/WorkerThreadProject'; + +interface ProjectManagerConstructorArgs { + /** + * Should each project run in its own dedicated worker thread or all run on the main thread + */ + threadingEnabled: boolean; +} /** * Manages all brighterscript projects for the language server @@ -16,17 +22,16 @@ export class ProjectManager { /** * Collection of all projects */ - public projects: Project[] = []; + public projects: LspProject[] = []; /** * A unique project counter to help distinguish log entries in lsp mode */ - private projectCounter = 0; + private static projectNumberSequence = 0; - private emitter = new EventEmitter(); - public on(eventName: 'critical-failure', handler: (data: { project: Project; message: string }) => void); - public on(eventName: 'flush-diagnostics', handler: (data: { project: Project }) => void); + public on(eventName: 'critical-failure', handler: (data: { project: LspProject; message: string }) => void); + public on(eventName: 'flush-diagnostics', handler: (data: { project: LspProject }) => void); public on(eventName: string, handler: (payload: any) => void) { this.emitter.on(eventName, handler); return () => { @@ -34,13 +39,14 @@ export class ProjectManager { }; } - private emit(eventName: 'critical-failure', data: { project: Project; message: string }); - private emit(eventName: 'flush-diagnostics', data: { project: Project }); + private emit(eventName: 'critical-failure', data: { project: LspProject; message: string }); + private emit(eventName: 'flush-diagnostics', data: { project: LspProject }); private async emit(eventName: string, data?) { //emit these events on next tick, otherwise they will be processed immediately which could cause issues await util.sleep(0); this.emitter.emit(eventName, data); } + private emitter = new EventEmitter(); private fileResolvers = [] as FileResolver[]; @@ -56,14 +62,15 @@ export class ProjectManager { * @param workspaceConfigs an array of workspaces */ public async syncProjects(workspaceConfigs: WorkspaceConfig[]) { - //build a list of unique projects + //build a list of unique projects across all workspace folders let projectConfigs = (await Promise.all( workspaceConfigs.map(async workspaceConfig => { const projectPaths = await this.getProjectPaths(workspaceConfig); return projectPaths.map(projectPath => ({ projectPath: s`${projectPath}`, workspaceFolder: s`${workspaceConfig}`, - excludePatterns: workspaceConfig.excludePatterns + excludePatterns: workspaceConfig.excludePatterns, + threadingEnabled: workspaceConfig.threadingEnabled })); }) )).flat(1); @@ -163,146 +170,55 @@ export class ProjectManager { /** * Remove a project from the language server */ - private removeProject(project: Project) { + private removeProject(project: LspProject) { const idx = this.projects.findIndex(x => x.projectPath === project?.projectPath); if (idx > -1) { this.projects.splice(idx, 1); } - project?.builder?.dispose(); + project?.dispose(); } - private async createProject(config: ProjectConfig) { - config.workspaceFolder ??= config.projectPath; - + /** + * Create a project for the given config + * @param config + * @returns a new project, or the existing project if one already exists with this config info + */ + private async createProject(config1: ProjectConfig) { //skip this project if we already have it - if (this.hasProject(config.projectPath)) { - return this.getProject(config.projectPath); - } - - let builder = new ProgramBuilder(); - - config.projectNumber ??= this.projectCounter++; - - builder.logger.prefix = `[prj${config.projectNumber}]`; - builder.logger.log(`Created project #${config.projectNumber} for: "${config.projectPath}"`); - - //flush diagnostics every time the program finishes validating - builder.plugins.add({ - name: 'bsc-language-server', - afterProgramValidate: () => { - this.emit('flush-diagnostics', { project: this.getProject(config) }); - } - } as CompilerPlugin); - - //prevent clearing the console on run...this isn't the CLI so we want to keep a full log of everything - builder.allowConsoleClearing = false; - - //register any external file resolvers - builder.addFileResolver(...this.fileResolvers); - - let configFilePath = await this.getBsconfigPath(config); - - let cwd = config.projectPath; - - //if the config file exists, use it and its folder as cwd - if (configFilePath && await util.pathExists(configFilePath)) { - cwd = path.dirname(configFilePath); - } else { - //config file doesn't exist...let `brighterscript` resolve the default way - configFilePath = undefined; + if (this.hasProject(config1.projectPath)) { + return this.getProject(config1.projectPath); } - const firstRunDeferred = new Deferred(); - - let newProject: Project = { - projectNumber: config.projectNumber, - builder: builder, - firstRunPromise: firstRunDeferred.promise, - projectPath: config.projectPath, - workspaceFolder: config.workspaceFolder, - isFirstRunComplete: false, - isFirstRunSuccessful: false, - configFilePath: configFilePath, - isStandaloneFileProject: false - }; - - this.projects.push(newProject); + let project: LspProject = config1.threadingEnabled + ? new WorkerThreadProject() + : new Project(); - try { - await builder.run({ - cwd: cwd, - project: configFilePath, - watch: false, - createPackage: false, - deploy: false, - copyToStaging: false, - showDiagnosticsInConsole: false - }); - newProject.isFirstRunComplete = true; - newProject.isFirstRunSuccessful = true; - firstRunDeferred.resolve(); - } catch (e) { - builder.logger.error(e); - firstRunDeferred.reject(e); - newProject.isFirstRunComplete = true; - newProject.isFirstRunSuccessful = false; - } - //if we found a deprecated brsconfig.json, add a diagnostic warning the user - if (configFilePath && path.basename(configFilePath) === 'brsconfig.json') { - builder.addDiagnostic(configFilePath, { - ...DiagnosticMessages.brsConfigJsonIsDeprecated(), - range: util.createRange(0, 0, 0, 0) - }); - this.emit('flush-diagnostics', { project: newProject }); - } - return newProject; - } - - private async getBsconfigPath(config: ProjectConfig) { - - let configFilePath: string; - //if there's a setting, we need to find the file or show error if it can't be found - if (config?.bsconfigPath) { - configFilePath = path.resolve(config.projectPath, config.bsconfigPath); - if (await util.pathExists(configFilePath)) { - return configFilePath; - } else { - this.emit('critical-failure', { - message: `Cannot find config file specified in user / workspace settings at '${configFilePath}'`, - project: this.getProject(config) - }); - } - } - - //the rest of these require a projectPath, so return early if we don't have one - if (!config?.projectPath) { - return undefined; - } - - //default to config file path found in the root of the workspace - configFilePath = s`${config.projectPath}/bsconfig.json`; - if (await util.pathExists(configFilePath)) { - return configFilePath; - } - - //look for the deprecated `brsconfig.json` file - configFilePath = s`${config.projectPath}/brsconfig.json`; - if (await util.pathExists(configFilePath)) { - return configFilePath; - } - - //no config file could be found - return undefined; + await project.activate({ + projectPath: config1.projectPath, + workspaceFolder: config1.workspaceFolder, + projectNumber: config1.projectNumber ?? ProjectManager.projectNumberSequence++ + }); } } -interface WorkspaceConfig { +export interface WorkspaceConfig { + /** + * Absolute path to the folder where the workspace resides + */ workspaceFolder: string; + /** + * A list of glob patterns used to _exclude_ files from various bsconfig searches + */ excludePatterns?: string[]; /** * Path to a bsconfig that should be used instead of the auto-discovery algorithm. If this is present, no bsconfig discovery should be used. and an error should be emitted if this file is missing */ bsconfigPath?: string; + /** + * Should the projects in this workspace be run in their own dedicated worker threads, or all run on the main thread + * TODO - is there a better name for this? + */ + threadingEnabled?: boolean; } interface ProjectConfig { @@ -326,4 +242,9 @@ interface ProjectConfig { * Path to a bsconfig that should be used instead of the auto-discovery algorithm. If this is present, no bsconfig discovery should be used. and an error should be emitted if this file is missing */ bsconfigPath?: string; + /** + * Should this project run in its own dedicated worker thread + * TODO - is there a better name for this? + */ + threadingEnabled?: boolean; } diff --git a/src/lsp/worker/MessageHandler.spec.ts b/src/lsp/worker/MessageHandler.spec.ts new file mode 100644 index 000000000..7dcbfbc89 --- /dev/null +++ b/src/lsp/worker/MessageHandler.spec.ts @@ -0,0 +1,40 @@ +import { MessageChannel } from 'worker_threads'; +import { MessageHandler } from './MessageHandler'; +import { expect } from '../../chai-config.spec'; + +describe('MessageHandler', () => { + let server: MessageHandler; + let client: MessageHandler; + let channel: MessageChannel; + + beforeEach(() => { + channel = new MessageChannel(); + }); + + afterEach(() => { + server?.dispose(); + client?.dispose(); + channel.port1.close(); + channel.port2.close(); + }); + + it('serializes an error when present', async () => { + let server = new MessageHandler({ + port: channel.port1, + onRequest: (request) => { + server.sendResponse(request, { + error: new Error('Crash') + }); + } + }); + let client = new MessageHandler({ port: channel.port2 }); + let error: Error; + try { + await client.sendRequest('doSomething'); + } catch (e) { + error = e as any; + } + expect(error).to.exist; + expect(error).instanceof(Error); + }); +}); diff --git a/src/lsp/worker/MessageHandler.ts b/src/lsp/worker/MessageHandler.ts new file mode 100644 index 000000000..b2b37cd12 --- /dev/null +++ b/src/lsp/worker/MessageHandler.ts @@ -0,0 +1,176 @@ +import type { MessagePort, parentPort } from 'worker_threads'; +import * as EventEmitter from 'eventemitter3'; +import type { DisposableLike } from '../../interfaces'; +import util from '../../util'; + +interface PseudoMessagePort { + on: (name: 'message', cb: (message: any) => any) => any; + postMessage: typeof parentPort['postMessage']; +} + +export class MessageHandler { + constructor( + options: { + name?: string; + port: PseudoMessagePort; + onRequest?: (message: WorkerMessage) => any; + onResponse?: (message: WorkerMessage) => any; + onUpdate?: (message: WorkerMessage) => any; + } + ) { + this.name = options?.name; + this.port = options?.port; + const listener = (message: WorkerMessage) => { + switch (message.type) { + case 'request': + options?.onRequest?.(message); + break; + case 'response': + options?.onResponse?.(message); + this.emitter.emit(`${message.type}-${message.id}`, message); + break; + case 'update': + options?.onUpdate?.(message); + break; + } + }; + options?.port.on('message', listener); + + this.disposables.push( + this.emitter.removeAllListeners.bind(this.emitter), + () => (options?.port as MessagePort).off('message', listener) + ); + } + + /** + * An optional name to help with debugging this handler + */ + public readonly name: string; + + private port: PseudoMessagePort; + + private disposables: DisposableLike[] = []; + + private emitter = new EventEmitter(); + + /** + * Get the response with this ID + * @param id the ID of the response + * @returns the message + */ + private onResponse(id: number) { + return new Promise>((resolve) => { + this.emitter.once(`response-${id}`, (response) => { + resolve(response); + }); + }); + } + + /** + * A unique sequence for identifying messages + */ + private idSequence = 0; + + /** + * Send a request to the worker, and wait for a response. + * @param name the name of the request + * @param options the request options + */ + public async sendRequest(name: string, options?: { data: any; id?: number }) { + const request: WorkerMessage = { + type: 'request', + name: name, + data: options?.data, + id: options?.id ?? this.idSequence++ + }; + const responsePromise = this.onResponse(request.id); + this.port.postMessage(request); + const response = await responsePromise; + if (response.error) { + const error = this.objectToError(response.error); + (error as any)._response = response; + //throw the error so it causes a rejected promise (like we'd expect) + throw error; + } + return response; + } + + /** + * Send a request to the worker, and wait for a response. + * @param request the request we are responding to + * @param response the data to be sent as the response + */ + public sendResponse(request: WorkerMessage, options?: { data: any } | { error: Error } | undefined) { + const response: WorkerResponse = { + name: request.name, + type: 'response', + id: request.id + }; + if ('error' in options) { + //hack: turn the error into a plain json object + response.error = this.errorToObject(options.error); + } else if ('data' in options) { + response.data = options.data; + } + this.port.postMessage(response); + } + + /** + * Send a request to the worker, and wait for a response. + * @param name the name of the request + * @param options options for the update + */ + public sendUpdate(name: string, options?: { data?: any; id?: number }) { + let update: WorkerMessage = { + name: name, + data: options?.data, + type: 'update', + id: options?.id ?? this.idSequence++ + }; + this.port.postMessage(update); + } + + /** + * Convert an Error object into a plain object so it can be serialized + * @param error the error to object-ify + * @returns an object version of an error + */ + private errorToObject(error: Error) { + return { + name: error.name, + message: error.message, + stack: error.stack, + cause: (error.cause as any)?.message && (error.cause as any)?.stack ? this.errorToObject(error.cause as any) : error.cause + }; + } + + /** + * Turn an object with an error structure into a proper error + * @param error the error (in object form) to turn into a proper Error item + */ + private objectToError(error: Error) { + let result = new Error(); + result.name = error.name; + result.message = error.message; + result.stack = error.stack; + result.cause = (error.cause as any)?.message && (error.cause as any)?.stack ? this.objectToError(error.cause as any) : error.cause; + return result; + } + + public dispose() { + util.applyDispose(this.disposables); + } +} + +export interface WorkerMessage { + id: number; + type: 'request' | 'response' | 'update'; + name: string; + data?: T; +} +export interface WorkerResponse extends WorkerMessage { + /** + * An error occurred on the remote side. There will be no `.data` value + */ + error?: Error; +} diff --git a/src/lsp/worker/WorkerThreadProject.spec.ts b/src/lsp/worker/WorkerThreadProject.spec.ts new file mode 100644 index 000000000..14aa31418 --- /dev/null +++ b/src/lsp/worker/WorkerThreadProject.spec.ts @@ -0,0 +1,51 @@ +import { expect } from 'chai'; +import { tempDir, rootDir, expectDiagnosticsAsync } from '../../testHelpers.spec'; +import * as fsExtra from 'fs-extra'; +import { createSandbox } from 'sinon'; +import { standardizePath as s } from '../../util'; +import { WorkerThreadProject } from './WorkerThreadProject'; +import { DiagnosticMessages } from '../../DiagnosticMessages'; + +const sinon = createSandbox(); + +describe.only('WorkerThreadProject', () => { + let project: WorkerThreadProject; + + beforeEach(() => { + project?.dispose(); + project = new WorkerThreadProject(); + fsExtra.emptyDirSync(tempDir); + }); + + afterEach(() => { + fsExtra.emptyDirSync(tempDir); + project?.dispose(); + }); + + describe.only('activate', () => { + it('finds bsconfig.json at root', async () => { + fsExtra.outputFileSync(`${rootDir}/bsconfig.json`, ''); + await project.activate({ + projectPath: rootDir + }); + expect(project.configFilePath).to.eql(s`${rootDir}/bsconfig.json`); + }); + + it.only('shows diagnostics after running', async () => { + fsExtra.outputFileSync(`${rootDir}/source/main.brs`, ` + sub main() + print varNotThere + end sub + `); + + await project.activate({ + projectPath: rootDir, + projectNumber: 1 + }); + + await expectDiagnosticsAsync(project, [ + DiagnosticMessages.cannotFindName('varNotThere').message + ]); + }); + }); +}); diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts new file mode 100644 index 000000000..135274b7f --- /dev/null +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -0,0 +1,112 @@ +import * as EventEmitter from 'eventemitter3'; +import { Worker } from 'worker_threads'; +import type { WorkerMessage } from './MessageHandler'; +import { MessageHandler } from './MessageHandler'; +import util from '../../util'; +import type { LspDiagnostic } from '../LspProject'; +import { type ActivateOptions, type LspProject } from '../LspProject'; +import { isMainThread, parentPort, workerData } from 'worker_threads'; +import { WorkerThreadProjectRunner } from './WorkerThreadProjectRunner'; +import type { Project } from '../Project'; + +//if this script us running in a Worker, run +if (!isMainThread) { + const runner = new WorkerThreadProjectRunner(); + runner.run(parentPort); +} + +export class WorkerThreadProject implements LspProject { + + public async activate(options: ActivateOptions) { + this.projectPath = options.projectPath; + this.workspaceFolder = options.workspaceFolder; + this.projectNumber = options.projectNumber; + this.configFilePath = options.configFilePath; + + //start the worker thread + this.worker = new Worker( + __filename, + { + ...options, + //wire up ts-node if we're running in ts-node + execArgv: /\.ts$/i.test(__filename) ? ['--require', 'ts-node/register'] : undefined + } + ); + this.messageHandler = new MessageHandler({ + port: this.worker, + onRequest: this.processRequest.bind(this), + onUpdate: this.processUpdate.bind(this) + }); + + await this.messageHandler.sendRequest('activate'); + } + + public async getDiagnostics() { + const diagnostics = await this.messageHandler.sendRequest('getDiagnostics'); + return diagnostics.data; + } + + public dispose() { + //terminate the worker thread. we don't need to wait for it since this is immediate + this.worker.terminate(); + this.messageHandler.dispose(); + this.emitter.removeAllListeners(); + } + + /** + * Handles request/response/update messages from the worker thread + */ + private messageHandler: MessageHandler; + + private processRequest(request: WorkerMessage) { + + } + + private processUpdate(update: WorkerMessage) { + + } + + /** + * The worker thread where the actual project will execute + */ + private worker: Worker; + + /** + * The path to where the project resides + */ + public projectPath: string; + + /** + * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging + */ + public projectNumber: number; + + /** + * The path to the workspace where this project resides. A workspace can have multiple projects (by adding a bsconfig.json to each folder). + * Defaults to `.projectPath` if not set + */ + public workspaceFolder: string; + + /** + * Path to a bsconfig.json file that will be used for this project + */ + public configFilePath?: string; + + public on(eventName: 'critical-failure', handler: (data: { project: Project; message: string }) => void); + public on(eventName: 'flush-diagnostics', handler: (data: { project: Project }) => void); + public on(eventName: string, handler: (payload: any) => void) { + this.emitter.on(eventName, handler); + return () => { + this.emitter.removeListener(eventName, handler); + }; + } + + private emit(eventName: 'critical-failure', data: { project: Project; message: string }); + private emit(eventName: 'flush-diagnostics', data: { project: Project }); + private async emit(eventName: string, data?) { + //emit these events on next tick, otherwise they will be processed immediately which could cause issues + await util.sleep(0); + this.emitter.emit(eventName, data); + } + private emitter = new EventEmitter(); +} diff --git a/src/lsp/worker/WorkerThreadProjectRunner.ts b/src/lsp/worker/WorkerThreadProjectRunner.ts new file mode 100644 index 000000000..e8df90d7d --- /dev/null +++ b/src/lsp/worker/WorkerThreadProjectRunner.ts @@ -0,0 +1,32 @@ +import { Project } from '../Project'; +import type { WorkerMessage } from './MessageHandler'; +import { MessageHandler } from './MessageHandler'; +import type { MessagePort } from 'worker_threads'; + +/** + * Runner logic for Running a Project in a worker thread. + */ +export class WorkerThreadProjectRunner { + public run(parentPort: MessagePort) { + //make a new instance of the project (which is the same way we run it in the main thread). + const project = new Project(); + const messageHandler = new MessageHandler({ + port: parentPort, + onRequest: async (request: WorkerMessage) => { + try { + //only the LspProject interface method names will be passed as request names, so just call those functions on the Project class directly + let responseData = await project[request.name](...request.data ?? []); + messageHandler.sendResponse(request, { data: responseData }); + + //we encountered a runtime crash. Pass that error along as the response to this request + } catch (e) { + const error: Error = e as any; + messageHandler.sendResponse(request, { error: error }); + } + }, + onUpdate: (update) => { + + } + }); + } +} diff --git a/src/testHelpers.spec.ts b/src/testHelpers.spec.ts index a865fd74b..0ea415a7b 100644 --- a/src/testHelpers.spec.ts +++ b/src/testHelpers.spec.ts @@ -21,6 +21,7 @@ export const stagingDir = s`${tempDir}/stagingDir`; export const trim = undent; type DiagnosticCollection = { getDiagnostics(): Array } | { diagnostics: Diagnostic[] } | Diagnostic[]; +type DiagnosticCollectionAsync = DiagnosticCollection | { getDiagnostics(): Promise> }; function getDiagnostics(arg: DiagnosticCollection): BsDiagnostic[] { if (Array.isArray(arg)) { @@ -101,6 +102,17 @@ function cloneDiagnostic(actualDiagnosticInput: BsDiagnostic, expectedDiagnostic return actualDiagnostic; } +/** + * Ensure the DiagnosticCollection exactly contains the data from expected list. + * @param arg - any object that contains diagnostics (such as `Program`, `Scope`, or even an array of diagnostics) + * @param expected an array of expected diagnostics. if it's a string, assume that's a diagnostic error message + */ +export async function expectDiagnosticsAsync(arg: DiagnosticCollectionAsync, expected: Array) { + expectDiagnostics( + await Promise.resolve(getDiagnostics(arg as any)), + expected + ); +} /** * Ensure the DiagnosticCollection exactly contains the data from expected list. diff --git a/src/util.ts b/src/util.ts index 4253f72ad..69fc1ff5d 100644 --- a/src/util.ts +++ b/src/util.ts @@ -4,12 +4,12 @@ import type { ParseError } from 'jsonc-parser'; import { parse as parseJsonc, printParseErrorCode } from 'jsonc-parser'; import * as path from 'path'; import { rokuDeploy, DefaultFiles, standardizePath as rokuDeployStandardizePath } from 'roku-deploy'; -import type { Diagnostic, Position, Range, Location } from 'vscode-languageserver'; +import type { Diagnostic, Position, Range, Location, Disposable } from 'vscode-languageserver'; import { URI } from 'vscode-uri'; import * as xml2js from 'xml2js'; import type { BsConfig } from './BsConfig'; import { DiagnosticMessages } from './DiagnosticMessages'; -import type { CallableContainer, BsDiagnostic, FileReference, CallableContainerMap, CompilerPluginFactory, CompilerPlugin, ExpressionInfo } from './interfaces'; +import type { CallableContainer, BsDiagnostic, FileReference, CallableContainerMap, CompilerPluginFactory, CompilerPlugin, ExpressionInfo, DisposableLike } from './interfaces'; import { BooleanType } from './types/BooleanType'; import { DoubleType } from './types/DoubleType'; import { DynamicType } from './types/DynamicType'; @@ -1280,7 +1280,7 @@ export class Util { //filter out null relatedInformation items }).filter(x => x), code: diagnostic.code, - source: 'brs' + source: diagnostic.source ?? 'brs' }; } @@ -1497,6 +1497,20 @@ export class Util { }]); } } + + /** + * Execute dispose for a series of disposable items + * @param disposables a list of functions or disposables + */ + public applyDispose(disposables: DisposableLike[]) { + for (const disposable of disposables ?? []) { + if (typeof disposable === 'function') { + disposable(); + } else { + disposable?.dispose?.(); + } + } + } } /** From 2b727ff83d116db72aadf5bec16839145122aa87 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sat, 23 Dec 2023 01:00:22 -0500 Subject: [PATCH 004/119] Fix workerThreadProject tests --- src/lsp/Project.ts | 10 ++++- src/lsp/worker/MessageHandler.ts | 12 +++--- src/lsp/worker/WorkerPool.ts | 48 +++++++++++++++++++++ src/lsp/worker/WorkerThreadProject.spec.ts | 31 +++++++------ src/lsp/worker/WorkerThreadProject.ts | 34 +++++++++------ src/lsp/worker/WorkerThreadProjectRunner.ts | 1 + 6 files changed, 101 insertions(+), 35 deletions(-) create mode 100644 src/lsp/worker/WorkerPool.ts diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 720bf5012..bfbc1a7bb 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -9,7 +9,13 @@ import { URI } from 'vscode-uri'; export class Project implements LspProject { + /** + * Activates this project. Every call to `activate` should completely reset the project, clear all used ram and start from scratch. + * @param options + */ public async activate(options: ActivateOptions) { + this.dispose(); + this.projectPath = options.projectPath; this.workspaceFolder = options.workspaceFolder; this.projectNumber = options.projectNumber; @@ -78,12 +84,14 @@ export class Project implements LspProject { } public dispose() { + this.builder?.dispose(); + this.emitter?.removeAllListeners(); } /** * Manages the BrighterScript program. The main interface into the compiler/validator */ - private builder = new ProgramBuilder(); + private builder: ProgramBuilder; /** * The path to where the project resides diff --git a/src/lsp/worker/MessageHandler.ts b/src/lsp/worker/MessageHandler.ts index b2b37cd12..39ba6471a 100644 --- a/src/lsp/worker/MessageHandler.ts +++ b/src/lsp/worker/MessageHandler.ts @@ -76,11 +76,11 @@ export class MessageHandler { * @param name the name of the request * @param options the request options */ - public async sendRequest(name: string, options?: { data: any; id?: number }) { + public async sendRequest(name: string, options?: { data: any[]; id?: number }) { const request: WorkerMessage = { type: 'request', name: name, - data: options?.data, + data: options?.data ?? [], id: options?.id ?? this.idSequence++ }; const responsePromise = this.onResponse(request.id); @@ -102,8 +102,8 @@ export class MessageHandler { */ public sendResponse(request: WorkerMessage, options?: { data: any } | { error: Error } | undefined) { const response: WorkerResponse = { - name: request.name, type: 'response', + name: request.name, id: request.id }; if ('error' in options) { @@ -120,11 +120,11 @@ export class MessageHandler { * @param name the name of the request * @param options options for the update */ - public sendUpdate(name: string, options?: { data?: any; id?: number }) { + public sendUpdate(name: string, options?: { data?: any[]; id?: number }) { let update: WorkerMessage = { - name: name, - data: options?.data, type: 'update', + name: name, + data: options?.data ?? [], id: options?.id ?? this.idSequence++ }; this.port.postMessage(update); diff --git a/src/lsp/worker/WorkerPool.ts b/src/lsp/worker/WorkerPool.ts new file mode 100644 index 000000000..da765b7a0 --- /dev/null +++ b/src/lsp/worker/WorkerPool.ts @@ -0,0 +1,48 @@ +import type { Worker } from 'worker_threads'; + +export class WorkerPool { + constructor( + private factory: () => Worker + ) { + + } + + private workers: Worker[] = []; + + /** + * Ensure that there are ${count} workers available in the pool + * @param count + */ + public preload(count: number) { + while (this.workers.length < count) { + this.workers.push( + this.getWorker() + ); + } + } + + /** + * Get a worker from the pool, or create a new one if none are available + * @returns a worker + */ + public getWorker() { + return this.workers.pop() ?? this.factory(); + } + + /** + * Give the worker back to the pool so it can be used by someone else + * @param worker the worker + */ + public releaseWorker(worker: Worker) { + this.workers.push(worker); + } + + /** + * Shut down all active worker pools + */ + public dispose() { + for (const worker of this.workers) { + worker.terminate(); + } + } +} diff --git a/src/lsp/worker/WorkerThreadProject.spec.ts b/src/lsp/worker/WorkerThreadProject.spec.ts index 14aa31418..87e593e4d 100644 --- a/src/lsp/worker/WorkerThreadProject.spec.ts +++ b/src/lsp/worker/WorkerThreadProject.spec.ts @@ -1,15 +1,13 @@ -import { expect } from 'chai'; import { tempDir, rootDir, expectDiagnosticsAsync } from '../../testHelpers.spec'; import * as fsExtra from 'fs-extra'; -import { createSandbox } from 'sinon'; -import { standardizePath as s } from '../../util'; -import { WorkerThreadProject } from './WorkerThreadProject'; +import { WorkerThreadProject, workerPool } from './WorkerThreadProject'; import { DiagnosticMessages } from '../../DiagnosticMessages'; -const sinon = createSandbox(); - describe.only('WorkerThreadProject', () => { let project: WorkerThreadProject; + before(() => { + workerPool.preload(1); + }); beforeEach(() => { project?.dispose(); @@ -22,16 +20,21 @@ describe.only('WorkerThreadProject', () => { project?.dispose(); }); - describe.only('activate', () => { - it('finds bsconfig.json at root', async () => { - fsExtra.outputFileSync(`${rootDir}/bsconfig.json`, ''); - await project.activate({ - projectPath: rootDir - }); - expect(project.configFilePath).to.eql(s`${rootDir}/bsconfig.json`); + after(() => { + //shut down all the worker threads after we're finished with all the tests + workerPool.dispose(); + }); + + it('wake up the worker thread', async function test() { + this.timeout(20_000); + await project.activate({ + projectPath: rootDir, + projectNumber: 1 }); + }); - it.only('shows diagnostics after running', async () => { + describe('activate', () => { + it('shows diagnostics after running', async () => { fsExtra.outputFileSync(`${rootDir}/source/main.brs`, ` sub main() print varNotThere diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 135274b7f..a1dcd6f57 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -5,9 +5,20 @@ import { MessageHandler } from './MessageHandler'; import util from '../../util'; import type { LspDiagnostic } from '../LspProject'; import { type ActivateOptions, type LspProject } from '../LspProject'; -import { isMainThread, parentPort, workerData } from 'worker_threads'; +import { isMainThread, parentPort } from 'worker_threads'; import { WorkerThreadProjectRunner } from './WorkerThreadProjectRunner'; import type { Project } from '../Project'; +import { WorkerPool } from './WorkerPool'; + +export const workerPool = new WorkerPool(() => { + return new Worker( + __filename, + { + //wire up ts-node if we're running in ts-node + execArgv: /\.ts$/i.test(__filename) ? ['--require', 'ts-node/register'] : undefined + } + ); +}); //if this script us running in a Worker, run if (!isMainThread) { @@ -23,22 +34,15 @@ export class WorkerThreadProject implements LspProject { this.projectNumber = options.projectNumber; this.configFilePath = options.configFilePath; - //start the worker thread - this.worker = new Worker( - __filename, - { - ...options, - //wire up ts-node if we're running in ts-node - execArgv: /\.ts$/i.test(__filename) ? ['--require', 'ts-node/register'] : undefined - } - ); + // start a new worker thread or get an unused existing thread + this.worker = workerPool.getWorker(); this.messageHandler = new MessageHandler({ + name: 'MainThread', port: this.worker, onRequest: this.processRequest.bind(this), onUpdate: this.processUpdate.bind(this) }); - - await this.messageHandler.sendRequest('activate'); + await this.messageHandler.sendRequest('activate', { data: [options] }); } public async getDiagnostics() { @@ -47,8 +51,10 @@ export class WorkerThreadProject implements LspProject { } public dispose() { - //terminate the worker thread. we don't need to wait for it since this is immediate - this.worker.terminate(); + //restore the worker back to the worker pool so it can be used again + if (this.worker) { + workerPool.releaseWorker(this.worker); + } this.messageHandler.dispose(); this.emitter.removeAllListeners(); } diff --git a/src/lsp/worker/WorkerThreadProjectRunner.ts b/src/lsp/worker/WorkerThreadProjectRunner.ts index e8df90d7d..f9b2858d9 100644 --- a/src/lsp/worker/WorkerThreadProjectRunner.ts +++ b/src/lsp/worker/WorkerThreadProjectRunner.ts @@ -11,6 +11,7 @@ export class WorkerThreadProjectRunner { //make a new instance of the project (which is the same way we run it in the main thread). const project = new Project(); const messageHandler = new MessageHandler({ + name: 'WorkerThread', port: parentPort, onRequest: async (request: WorkerMessage) => { try { From 29bb2df66f07eef82b3cba3d40002f2fe90ce244 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sun, 24 Dec 2023 07:45:28 -0500 Subject: [PATCH 005/119] Fixing lots of broken tests --- src/lsp/Project.spec.ts | 85 +++++-------- src/lsp/Project.ts | 32 +++-- src/lsp/ProjectManager.spec.ts | 138 +++++++++------------ src/lsp/ProjectManager.ts | 18 +-- src/lsp/worker/WorkerPool.spec.ts | 5 + src/lsp/worker/WorkerPool.ts | 40 ++++-- src/lsp/worker/WorkerThreadProject.spec.ts | 37 +++--- src/lsp/worker/WorkerThreadProject.ts | 20 ++- 8 files changed, 177 insertions(+), 198 deletions(-) create mode 100644 src/lsp/worker/WorkerPool.spec.ts diff --git a/src/lsp/Project.spec.ts b/src/lsp/Project.spec.ts index f4a735844..13401cb94 100644 --- a/src/lsp/Project.spec.ts +++ b/src/lsp/Project.spec.ts @@ -3,25 +3,43 @@ import { tempDir, rootDir, expectDiagnosticsAsync } from '../testHelpers.spec'; import * as fsExtra from 'fs-extra'; import { standardizePath as s } from '../util'; import { Deferred } from '../deferred'; -import { createSandbox } from 'sinon'; import { DiagnosticMessages } from '../DiagnosticMessages'; -import { ProgramBuilder } from '..'; import { Project } from './Project'; +import { createSandbox } from 'sinon'; const sinon = createSandbox(); -describe('ProjectManager', () => { +describe.only('Project', () => { let project: Project; beforeEach(() => { + sinon.restore(); project = new Project(); fsExtra.emptyDirSync(tempDir); }); afterEach(() => { + sinon.restore(); fsExtra.emptyDirSync(tempDir); project.dispose(); }); + describe('on', () => { + it('emits events', async () => { + const stub = sinon.stub(); + const off = project.on('flush-diagnostics', stub); + await project['emit']('flush-diagnostics', { project: project }); + expect(stub.callCount).to.eql(1); + + await project['emit']('flush-diagnostics', { project: project }); + expect(stub.callCount).to.eql(2); + + off(); + + await project['emit']('flush-diagnostics', { project: project }); + expect(stub.callCount).to.eql(2); + }); + }); + describe('activate', () => { it('finds bsconfig.json at root', async () => { fsExtra.outputFileSync(`${rootDir}/bsconfig.json`, ''); @@ -31,7 +49,7 @@ describe('ProjectManager', () => { expect(project.configFilePath).to.eql(s`${rootDir}/bsconfig.json`); }); - it('shows diagnostics after running', async () => { + it('produces diagnostics after running', async () => { fsExtra.outputFileSync(`${rootDir}/source/main.brs`, ` sub main() print varNotThere @@ -50,51 +68,35 @@ describe('ProjectManager', () => { describe('createProject', () => { it('uses given projectNumber', async () => { - await manager['createProject']({ + await project.activate({ projectPath: rootDir, - workspaceFolder: rootDir, - projectNumber: 3 + projectNumber: 123 }); - expect(manager.projects[0].projectNumber).to.eql(3); + expect(project.projectNumber).to.eql(123); }); it('warns about deprecated brsconfig.json', async () => { fsExtra.outputFileSync(`${rootDir}/subdir1/brsconfig.json`, ''); - const project = await manager['createProject']({ + await project.activate({ projectPath: rootDir, workspaceFolder: rootDir, - bsconfigPath: 'subdir1/brsconfig.json' + configFilePath: 'subdir1/brsconfig.json' }); await expectDiagnosticsAsync(project, [ DiagnosticMessages.brsConfigJsonIsDeprecated() ]); }); - - it('properly tracks a failed run', async () => { - //force a total crash - sinon.stub(ProgramBuilder.prototype, 'run').returns( - Promise.reject(new Error('Critical failure')) - ); - const project = await manager['createProject']({ - projectPath: rootDir, - workspaceFolder: rootDir, - bsconfigPath: 'subdir1/brsconfig.json' - }); - expect(project.isFirstRunComplete).to.be.true; - expect(project.isFirstRunSuccessful).to.be.false; - }); }); - describe('getBsconfigPath', () => { + describe('getConfigPath', () => { it('emits critical failure for missing file', async () => { const deferred = new Deferred(); - manager.on('critical-failure', (event) => { + project.on('critical-failure', (event) => { deferred.resolve(event.message); }); - await manager['getBsconfigPath']({ + await project['getConfigFilePath']({ projectPath: rootDir, - workspaceFolder: rootDir, - bsconfigPath: s`${rootDir}/bsconfig.json` + configFilePath: s`${rootDir}/bsconfig.json` }); expect( (await deferred.promise).startsWith('Cannot find config file') @@ -104,33 +106,14 @@ describe('ProjectManager', () => { it('finds brsconfig.json', async () => { fsExtra.outputFileSync(`${rootDir}/brsconfig.json`, ''); expect( - await manager['getBsconfigPath']({ - projectPath: rootDir, - workspaceFolder: rootDir + await project['getConfigFilePath']({ + projectPath: rootDir }) ).to.eql(s`${rootDir}/brsconfig.json`); }); - it('does not crash on undefined', async () => { - await manager['getBsconfigPath'](undefined); - }); - }); - - describe('removeProject', () => { - it('handles undefined', async () => { - manager['removeProject'](undefined); - await manager.syncProjects([{ - workspaceFolder: rootDir - }]); - manager['removeProject'](undefined); - }); - - it('does not crash when removing project that is not there', () => { - manager['removeProject']({ - projectPath: rootDir, - dispose: () => { } - } as any); + await project['getConfigFilePath'](undefined); }); }); }); diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index bfbc1a7bb..f40f4fb65 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -47,19 +47,15 @@ export class Project implements LspProject { //TODO handle in-memory file stuff // builder.addFileResolver(...this.fileResolvers); - try { - await this.builder.run({ - cwd: cwd, - project: this.configFilePath, - watch: false, - createPackage: false, - deploy: false, - copyToStaging: false, - showDiagnosticsInConsole: false - }); - } catch (e) { - this.builder.logger.error(e); - } + await this.builder.run({ + cwd: cwd, + project: this.configFilePath, + watch: false, + createPackage: false, + deploy: false, + copyToStaging: false, + showDiagnosticsInConsole: false + }); //if we found a deprecated brsconfig.json, add a diagnostic warning the user if (this.configFilePath && path.basename(this.configFilePath) === 'brsconfig.json') { @@ -83,11 +79,6 @@ export class Project implements LspProject { ); } - public dispose() { - this.builder?.dispose(); - this.emitter?.removeAllListeners(); - } - /** * Manages the BrighterScript program. The main interface into the compiler/validator */ @@ -173,4 +164,9 @@ export class Project implements LspProject { this.emitter.emit(eventName, data); } private emitter = new EventEmitter(); + + public dispose() { + this.builder?.dispose(); + this.emitter.removeAllListeners(); + } } diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index fd393c3ce..cf364fb47 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -1,54 +1,48 @@ import { expect } from 'chai'; import { ProjectManager } from './ProjectManager'; -import { tempDir, rootDir, expectDiagnostics, expectDiagnosticsAsync } from '../testHelpers.spec'; +import { tempDir, rootDir } from '../testHelpers.spec'; import * as fsExtra from 'fs-extra'; import { standardizePath as s } from '../util'; -import { Deferred } from '../deferred'; import { createSandbox } from 'sinon'; -import { DiagnosticMessages } from '../DiagnosticMessages'; -import { ProgramBuilder } from '..'; +import { Project } from './Project'; +import { WorkerThreadProject } from './worker/WorkerThreadProject'; +import { wakeWorkerThread } from './worker/WorkerThreadProject.spec'; const sinon = createSandbox(); -describe('ProjectManager', () => { +describe.only('ProjectManager', () => { let manager: ProjectManager; + before(async function warmUpWorker() { + this.timeout(20_000); + await wakeWorkerThread(); + }); + beforeEach(() => { manager = new ProjectManager(); fsExtra.emptyDirSync(tempDir); + sinon.restore(); }); afterEach(() => { fsExtra.emptyDirSync(tempDir); + sinon.restore(); }); - describe('addFileResolver', () => { - it('runs added resolvers', async () => { - const mock = sinon.mock(); - manager.addFileResolver(mock); - fsExtra.outputFileSync(`${rootDir}/source/main.brs`, ''); - await manager.syncProjects([{ - workspaceFolder: rootDir - }]); - expect(mock.called).to.be.true; - }); - }); + describe('on', () => { + it('emits events', async () => { + const stub = sinon.stub(); + const off = manager.on('flush-diagnostics', stub); + await manager['emit']('flush-diagnostics', { project: undefined }); + expect(stub.callCount).to.eql(1); - describe('events', () => { - it('emits flush-diagnostics after validate finishes', async () => { - const deferred = new Deferred(); - const disable = manager.on('flush-diagnostics', () => { - deferred.resolve(true); - }); - await manager.syncProjects([{ - workspaceFolder: rootDir - }]); - expect( - await deferred.promise - ).to.eql(true); + await manager['emit']('flush-diagnostics', { project: undefined }); + expect(stub.callCount).to.eql(2); + + off(); - //disable future events - disable(); + await manager['emit']('flush-diagnostics', { project: undefined }); + expect(stub.callCount).to.eql(2); }); }); @@ -165,6 +159,29 @@ describe('ProjectManager', () => { s`${rootDir}/subdir2` ]); }); + + it('spawns a worker thread when threading is enabled', async () => { + await manager.syncProjects([{ + workspaceFolder: rootDir, + threadingEnabled: true + }]); + expect(manager.projects[0]).instanceof(WorkerThreadProject); + }); + }); + + describe('getProject', () => { + it('uses .projectPath if param is not a string', async () => { + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expect( + manager['getProject']({ + projectPath: rootDir + }) + ).to.include({ + projectPath: rootDir + }); + }); }); describe('createProject', () => { @@ -188,61 +205,22 @@ describe('ProjectManager', () => { expect(manager.projects[0].projectNumber).to.eql(3); }); - it('warns about deprecated brsconfig.json', async () => { - fsExtra.outputFileSync(`${rootDir}/subdir1/brsconfig.json`, ''); - const project = await manager['createProject']({ - projectPath: rootDir, - workspaceFolder: rootDir, - bsconfigPath: 'subdir1/brsconfig.json' - }); - await expectDiagnosticsAsync(project, [ - DiagnosticMessages.brsConfigJsonIsDeprecated() - ]); - }); - it('properly tracks a failed run', async () => { //force a total crash - sinon.stub(ProgramBuilder.prototype, 'run').returns( + sinon.stub(Project.prototype, 'activate').returns( Promise.reject(new Error('Critical failure')) ); - const project = await manager['createProject']({ - projectPath: rootDir, - workspaceFolder: rootDir, - bsconfigPath: 'subdir1/brsconfig.json' - }); - expect(project.isFirstRunComplete).to.be.true; - expect(project.isFirstRunSuccessful).to.be.false; - }); - }); - - describe('getBsconfigPath', () => { - it('emits critical failure for missing file', async () => { - const deferred = new Deferred(); - manager.on('critical-failure', (event) => { - deferred.resolve(event.message); - }); - await manager['getBsconfigPath']({ - projectPath: rootDir, - workspaceFolder: rootDir, - bsconfigPath: s`${rootDir}/bsconfig.json` - }); - expect( - (await deferred.promise).startsWith('Cannot find config file') - ).to.be.true; - }); - - it('finds brsconfig.json', async () => { - fsExtra.outputFileSync(`${rootDir}/brsconfig.json`, ''); - expect( - await manager['getBsconfigPath']({ + let error; + try { + await manager['createProject']({ projectPath: rootDir, - workspaceFolder: rootDir - }) - ).to.eql(s`${rootDir}/brsconfig.json`); - }); - - it('does not crash on undefined', async () => { - await manager['getBsconfigPath'](undefined); + workspaceFolder: rootDir, + bsconfigPath: 'subdir1/brsconfig.json' + }); + } catch (e) { + error = e; + } + expect(error).to.include({ message: 'Critical failure' }); }); }); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 0deff9d84..38ef35c9a 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -2,18 +2,10 @@ import { standardizePath as s, util } from '../util'; import { rokuDeploy } from 'roku-deploy'; import * as path from 'path'; import * as EventEmitter from 'eventemitter3'; -import type { FileResolver } from '../interfaces'; import type { LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -interface ProjectManagerConstructorArgs { - /** - * Should each project run in its own dedicated worker thread or all run on the main thread - */ - threadingEnabled: boolean; -} - /** * Manages all brighterscript projects for the language server */ @@ -48,12 +40,6 @@ export class ProjectManager { } private emitter = new EventEmitter(); - private fileResolvers = [] as FileResolver[]; - - public addFileResolver(fileResolver: FileResolver) { - this.fileResolvers.push(fileResolver); - } - /** * Given a list of all desired projects, create any missing projects and destroy and projects that are no longer available * Treat workspaces that don't have a bsconfig.json as a project. @@ -193,6 +179,10 @@ export class ProjectManager { ? new WorkerThreadProject() : new Project(); + this.projects.push(project); + + //TODO subscribe to various events for this project + await project.activate({ projectPath: config1.projectPath, workspaceFolder: config1.workspaceFolder, diff --git a/src/lsp/worker/WorkerPool.spec.ts b/src/lsp/worker/WorkerPool.spec.ts new file mode 100644 index 000000000..fc03549bc --- /dev/null +++ b/src/lsp/worker/WorkerPool.spec.ts @@ -0,0 +1,5 @@ +import { wakeWorkerThread } from './WorkerThreadProject.spec'; + +describe('WorkerPool', () => { + +}); diff --git a/src/lsp/worker/WorkerPool.ts b/src/lsp/worker/WorkerPool.ts index da765b7a0..5ec920153 100644 --- a/src/lsp/worker/WorkerPool.ts +++ b/src/lsp/worker/WorkerPool.ts @@ -7,15 +7,22 @@ export class WorkerPool { } - private workers: Worker[] = []; + /** + * List of workers that are free to be used by a new task + */ + private freeWorkers: Worker[] = []; + /** + * List of all workers that we've ever created + */ + private allWorkers: Worker[] = []; /** * Ensure that there are ${count} workers available in the pool - * @param count + * @param count the number of total free workers that should exist when this function exits */ public preload(count: number) { - while (this.workers.length < count) { - this.workers.push( + while (this.freeWorkers.length < count) { + this.freeWorkers.push( this.getWorker() ); } @@ -26,7 +33,15 @@ export class WorkerPool { * @returns a worker */ public getWorker() { - return this.workers.pop() ?? this.factory(); + //we have no free workers. spin up a new one + if (this.freeWorkers.length === 0) { + const worker = this.factory(); + this.allWorkers.push(worker); + return worker; + } else { + //return an existing free worker + return this.freeWorkers.pop(); + } } /** @@ -34,15 +49,24 @@ export class WorkerPool { * @param worker the worker */ public releaseWorker(worker: Worker) { - this.workers.push(worker); + //add this worker back to the free workers list (if it's not already in there) + if (!this.freeWorkers.includes(worker)) { + this.freeWorkers.push(worker); + } } /** * Shut down all active worker pools */ public dispose() { - for (const worker of this.workers) { - worker.terminate(); + for (const worker of this.allWorkers) { + try { + worker.terminate(); + } catch (e) { + console.error(e); + } } + this.allWorkers = []; + this.freeWorkers = []; } } diff --git a/src/lsp/worker/WorkerThreadProject.spec.ts b/src/lsp/worker/WorkerThreadProject.spec.ts index 87e593e4d..f5ade3a2e 100644 --- a/src/lsp/worker/WorkerThreadProject.spec.ts +++ b/src/lsp/worker/WorkerThreadProject.spec.ts @@ -3,10 +3,28 @@ import * as fsExtra from 'fs-extra'; import { WorkerThreadProject, workerPool } from './WorkerThreadProject'; import { DiagnosticMessages } from '../../DiagnosticMessages'; -describe.only('WorkerThreadProject', () => { +export async function wakeWorkerThread() { + console.log('waking up a worker thread'); + const project = new WorkerThreadProject(); + try { + await project.activate({ + projectPath: rootDir, + projectNumber: 1 + }); + } finally { + project.dispose(); + } +} + +after(() => { + workerPool.dispose(); +}); + +describe('WorkerThreadProject', () => { let project: WorkerThreadProject; - before(() => { - workerPool.preload(1); + before(async function warmUpWorker() { + this.timeout(20_000); + await wakeWorkerThread(); }); beforeEach(() => { @@ -20,19 +38,6 @@ describe.only('WorkerThreadProject', () => { project?.dispose(); }); - after(() => { - //shut down all the worker threads after we're finished with all the tests - workerPool.dispose(); - }); - - it('wake up the worker thread', async function test() { - this.timeout(20_000); - await project.activate({ - projectPath: rootDir, - projectNumber: 1 - }); - }); - describe('activate', () => { it('shows diagnostics after running', async () => { fsExtra.outputFileSync(`${rootDir}/source/main.brs`, ` diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index a1dcd6f57..421fbbb62 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -50,15 +50,6 @@ export class WorkerThreadProject implements LspProject { return diagnostics.data; } - public dispose() { - //restore the worker back to the worker pool so it can be used again - if (this.worker) { - workerPool.releaseWorker(this.worker); - } - this.messageHandler.dispose(); - this.emitter.removeAllListeners(); - } - /** * Handles request/response/update messages from the worker thread */ @@ -99,7 +90,6 @@ export class WorkerThreadProject implements LspProject { public configFilePath?: string; public on(eventName: 'critical-failure', handler: (data: { project: Project; message: string }) => void); - public on(eventName: 'flush-diagnostics', handler: (data: { project: Project }) => void); public on(eventName: string, handler: (payload: any) => void) { this.emitter.on(eventName, handler); return () => { @@ -108,11 +98,19 @@ export class WorkerThreadProject implements LspProject { } private emit(eventName: 'critical-failure', data: { project: Project; message: string }); - private emit(eventName: 'flush-diagnostics', data: { project: Project }); private async emit(eventName: string, data?) { //emit these events on next tick, otherwise they will be processed immediately which could cause issues await util.sleep(0); this.emitter.emit(eventName, data); } private emitter = new EventEmitter(); + + public dispose() { + //move the worker back to the pool so it can be used again + if (this.worker) { + workerPool.releaseWorker(this.worker); + } + this.messageHandler.dispose(); + this.emitter.removeAllListeners(); + } } From 0e7a1f183ac50c46da0b5849a2cccd8d984c2f83 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sun, 24 Dec 2023 16:23:24 -0500 Subject: [PATCH 006/119] Fix some WorkerPool tests --- src/lsp/ProjectManager.spec.ts | 6 +- src/lsp/worker/WorkerPool.spec.ts | 66 +++++++++++++++++++++- src/lsp/worker/WorkerPool.ts | 15 +++-- src/lsp/worker/WorkerThreadProject.spec.ts | 14 +++-- src/lsp/worker/WorkerThreadProject.ts | 6 +- 5 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index cf364fb47..d095a64b8 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -6,15 +6,15 @@ import { standardizePath as s } from '../util'; import { createSandbox } from 'sinon'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import { wakeWorkerThread } from './worker/WorkerThreadProject.spec'; +import { wakeWorkerThreadPromise } from './worker/WorkerThreadProject.spec'; const sinon = createSandbox(); describe.only('ProjectManager', () => { let manager: ProjectManager; - before(async function warmUpWorker() { + before(async function workerThreadWarmup() { this.timeout(20_000); - await wakeWorkerThread(); + await wakeWorkerThreadPromise; }); beforeEach(() => { diff --git a/src/lsp/worker/WorkerPool.spec.ts b/src/lsp/worker/WorkerPool.spec.ts index fc03549bc..1365fac30 100644 --- a/src/lsp/worker/WorkerPool.spec.ts +++ b/src/lsp/worker/WorkerPool.spec.ts @@ -1,5 +1,67 @@ -import { wakeWorkerThread } from './WorkerThreadProject.spec'; +import { expect } from 'chai'; +import { WorkerPool } from './WorkerPool'; +import type { Worker } from 'worker_threads'; -describe('WorkerPool', () => { +describe.only('WorkerPool', () => { + let pool: WorkerPool; + let workers: Worker[] = [] as any; + beforeEach(() => { + workers = []; + //our factory will create empty objects. This prevents us from having to actually run threads. + pool = new WorkerPool(() => { + const worker = {} as Worker; + workers.push(worker); + return worker; + }); + }); + + describe('preload', () => { + it('ensures enough free workers have been created', () => { + expect(workers.length).to.eql(0); + + pool.preload(5); + expect(workers.length).to.eql(5); + + pool.preload(7); + expect(workers.length).to.eql(7); + }); + }); + + describe('releaseWorker', () => { + it('releases a worker back to the pool', () => { + const worker = pool.getWorker(); + expect(pool['freeWorkers']).lengthOf(0); + + pool.releaseWorker(worker); + expect(pool['freeWorkers']).lengthOf(1); + + //doesn't crash if we do the same thing again + pool.releaseWorker(worker); + expect(pool['freeWorkers']).lengthOf(1); + }); + }); + + describe('getWorker', () => { + it('creates a new worker when none exist', () => { + expect(pool['allWorkers']).to.be.empty; + expect(pool['freeWorkers']).to.be.empty; + const worker = pool.getWorker(); + expect(worker).to.eql(workers[0]); + expect(pool['allWorkers']).to.be.lengthOf(1); + //should be same instance + expect(pool['allWorkers'][0]).equals(workers[0]); + }); + }); + + describe('dispose', () => { + it('does not crash when worker.terminate() fails', () => { + const worker = pool.getWorker(); + worker['terminate'] = () => { + throw new Error('Test crash'); + }; + //should not throw error + pool.dispose(); + }); + }); }); diff --git a/src/lsp/worker/WorkerPool.ts b/src/lsp/worker/WorkerPool.ts index 5ec920153..6eb4223fc 100644 --- a/src/lsp/worker/WorkerPool.ts +++ b/src/lsp/worker/WorkerPool.ts @@ -23,11 +23,20 @@ export class WorkerPool { public preload(count: number) { while (this.freeWorkers.length < count) { this.freeWorkers.push( - this.getWorker() + this.createWorker() ); } } + /** + * Create a new worker + */ + private createWorker() { + const worker = this.factory(); + this.allWorkers.push(worker); + return worker; + } + /** * Get a worker from the pool, or create a new one if none are available * @returns a worker @@ -35,9 +44,7 @@ export class WorkerPool { public getWorker() { //we have no free workers. spin up a new one if (this.freeWorkers.length === 0) { - const worker = this.factory(); - this.allWorkers.push(worker); - return worker; + return this.createWorker(); } else { //return an existing free worker return this.freeWorkers.pop(); diff --git a/src/lsp/worker/WorkerThreadProject.spec.ts b/src/lsp/worker/WorkerThreadProject.spec.ts index f5ade3a2e..d4c2fe37c 100644 --- a/src/lsp/worker/WorkerThreadProject.spec.ts +++ b/src/lsp/worker/WorkerThreadProject.spec.ts @@ -2,6 +2,7 @@ import { tempDir, rootDir, expectDiagnosticsAsync } from '../../testHelpers.spec import * as fsExtra from 'fs-extra'; import { WorkerThreadProject, workerPool } from './WorkerThreadProject'; import { DiagnosticMessages } from '../../DiagnosticMessages'; +import { expect } from 'chai'; export async function wakeWorkerThread() { console.log('waking up a worker thread'); @@ -16,15 +17,17 @@ export async function wakeWorkerThread() { } } +export const wakeWorkerThreadPromise = wakeWorkerThread(); + after(() => { workerPool.dispose(); }); -describe('WorkerThreadProject', () => { +describe.only('WorkerThreadProject', () => { let project: WorkerThreadProject; - before(async function warmUpWorker() { + before(async function workerThreadWarmup() { this.timeout(20_000); - await wakeWorkerThread(); + await wakeWorkerThreadPromise; }); beforeEach(() => { @@ -50,8 +53,9 @@ describe('WorkerThreadProject', () => { projectPath: rootDir, projectNumber: 1 }); - - await expectDiagnosticsAsync(project, [ + const diagnostics = await project.getDiagnostics(); + expect(diagnostics).lengthOf(1); + await expectDiagnosticsAsync(diagnostics, [ DiagnosticMessages.cannotFindName('varNotThere').message ]); }); diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 421fbbb62..93ad5f32f 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -15,12 +15,16 @@ export const workerPool = new WorkerPool(() => { __filename, { //wire up ts-node if we're running in ts-node - execArgv: /\.ts$/i.test(__filename) ? ['--require', 'ts-node/register'] : undefined + execArgv: /\.ts$/i.test(__filename) + ? ['--require', 'ts-node/register'] + /* istanbul ignore next */ + : undefined } ); }); //if this script us running in a Worker, run +/* istanbul ignore next */ if (!isMainThread) { const runner = new WorkerThreadProjectRunner(); runner.run(parentPort); From 11b6c58e3831ed8249203c949f7b0b4f3f871c2b Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sun, 24 Dec 2023 16:37:19 -0500 Subject: [PATCH 007/119] Some language server structure tweaks --- src/LanguageServer.ts | 72 +++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index be2e15936..b563a73b2 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -50,16 +50,6 @@ import type { WorkspaceConfig } from './lsp/ProjectManager'; import { ProjectManager } from './lsp/ProjectManager'; export class LanguageServer { - constructor() { - this.projectManager = new ProjectManager(); - //anytime a project finishes validation, send diagnostics - this.projectManager.on('flush-diagnostics', () => { - void this.sendDiagnostics(); - }); - //allow the lsp to provide file contents - this.projectManager.addFileResolver(this.documentFileResolver.bind(this)); - } - private connection = undefined as Connection; /** @@ -94,10 +84,6 @@ export class LanguageServer { */ private documents = new TextDocuments(TextDocument); - private createConnection() { - return createConnection(ProposedFeatures.all); - } - private loggerSubscription: () => void; public validateThrottler = new Throttler(0); @@ -114,9 +100,17 @@ export class LanguageServer { //run the server public run() { + this.projectManager = new ProjectManager(); + //anytime a project finishes validation, send diagnostics + this.projectManager.on('flush-diagnostics', () => { + void this.sendDiagnostics(); + }); + //allow the lsp to provide file contents + //TODO handlet this... + // this.projectManager.addFileResolver(this.documentFileResolver.bind(this)); + // Create a connection for the server. The connection uses Node's IPC as a transport. - // Also include all preview / proposed LSP features. - this.connection = this.createConnection(); + this.establishConnection(); // Send the current status of the busyStatusTracker anytime it changes this.busyStatusTracker.on('change', (status) => { @@ -198,18 +192,6 @@ export class LanguageServer { this.connection.listen(); } - private busyStatusIndex = -1; - private sendBusyStatus(status: BusyStatus) { - this.busyStatusIndex = ++this.busyStatusIndex <= 0 ? 0 : this.busyStatusIndex; - - this.connection.sendNotification(NotificationName.busyStatus, { - status: status, - timestamp: Date.now(), - index: this.busyStatusIndex, - activeRuns: [...this.busyStatusTracker.activeRuns] - }); - } - /** * Called when the client starts initialization */ @@ -295,6 +277,32 @@ export class LanguageServer { } } + /** + * Establish a connection to the client if not already connected + */ + private establishConnection() { + if (!this.connection) { + this.connection = createConnection(ProposedFeatures.all); + } + } + + /** + * Send a new busy status notification to the client based on the current busy status + * @param status + */ + private sendBusyStatus(status: BusyStatus) { + this.busyStatusIndex = ++this.busyStatusIndex <= 0 ? 0 : this.busyStatusIndex; + + this.connection.sendNotification(NotificationName.busyStatus, { + status: status, + timestamp: Date.now(), + index: this.busyStatusIndex, + activeRuns: [...this.busyStatusTracker.activeRuns] + }); + } + private busyStatusIndex = -1; + + private initialProjectsCreated: Promise; /** @@ -484,14 +492,6 @@ export class LanguageServer { return newProject; } - private getProjects() { - let projects = this.projects.slice(); - for (let key in this.standaloneFileProjects) { - projects.push(this.standaloneFileProjects[key]); - } - return projects; - } - /** * Provide a list of completion items based on the current cursor position */ From 99d472d94a00496cbc58c4b20b64bd905aa3e186 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 25 Dec 2023 05:47:57 -0500 Subject: [PATCH 008/119] dynamically register on* lsp handlers. --- src/LanguageServer.ts | 963 +++++++++++++++++++++--------------------- 1 file changed, 487 insertions(+), 476 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index b563a73b2..30a490234 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -21,7 +21,11 @@ import type { SemanticTokens, SemanticTokensParams, TextDocumentChangeEvent, - Hover + Hover, + HandlerResult, + InitializeError, + InitializeResult, + InitializedParams } from 'vscode-languageserver/node'; import { SemanticTokensRequest, @@ -49,7 +53,11 @@ import { BusyStatusTracker } from './BusyStatusTracker'; import type { WorkspaceConfig } from './lsp/ProjectManager'; import { ProjectManager } from './lsp/ProjectManager'; -export class LanguageServer { +export class LanguageServer implements OnHandler { + + /** + * The language server protocol connection, used to send and receive all requests and responses + */ private connection = undefined as Connection; /** @@ -106,7 +114,7 @@ export class LanguageServer { void this.sendDiagnostics(); }); //allow the lsp to provide file contents - //TODO handlet this... + //TODO handle this... // this.projectManager.addFileResolver(this.documentFileResolver.bind(this)); // Create a connection for the server. The connection uses Node's IPC as a transport. @@ -122,13 +130,15 @@ export class LanguageServer { this.connection.tracer.log(text); }); - this.connection.onInitialize(this.onInitialize.bind(this)); - - this.connection.onInitialized(this.onInitialized.bind(this)); //eslint-disable-line - - this.connection.onDidChangeConfiguration(this.onDidChangeConfiguration.bind(this)); //eslint-disable-line + //bind all our on* methods that share the same name from connection + for (const name of Object.getOwnPropertyNames(LanguageServer.prototype)) { + if (/on+/.test(name) && typeof this.connection[name] === 'function') { + this.connection[name](this[name].bind(this)); + } + } - this.connection.onDidChangeWatchedFiles(this.onDidChangeWatchedFiles.bind(this)); //eslint-disable-line + //TODO switch to a more specific connection function call once they actually add it + this.connection.onRequest(SemanticTokensRequest.method, this.onFullSemanticTokens.bind(this)); // The content of a text document has changed. This event is emitted // when the text document is first opened, when its content has changed, @@ -139,52 +149,6 @@ export class LanguageServer { //whenever a document gets closed this.documents.onDidClose(this.onDocumentClose.bind(this)); - // This handler provides the initial list of the completion items. - this.connection.onCompletion(this.onCompletion.bind(this)); - - // This handler resolves additional information for the item selected in - // the completion list. - this.connection.onCompletionResolve(this.onCompletionResolve.bind(this)); - - this.connection.onHover(this.onHover.bind(this)); - - this.connection.onExecuteCommand(this.onExecuteCommand.bind(this)); - - this.connection.onDefinition(this.onDefinition.bind(this)); - - this.connection.onDocumentSymbol(this.onDocumentSymbol.bind(this)); - - this.connection.onWorkspaceSymbol(this.onWorkspaceSymbol.bind(this)); - - this.connection.onSignatureHelp(this.onSignatureHelp.bind(this)); - - this.connection.onReferences(this.onReferences.bind(this)); - - this.connection.onCodeAction(this.onCodeAction.bind(this)); - - //TODO switch to a more specific connection function call once they actually add it - this.connection.onRequest(SemanticTokensRequest.method, this.onFullSemanticTokens.bind(this)); - - /* - this.connection.onDidOpenTextDocument((params) => { - // A text document got opened in VSCode. - // params.uri uniquely identifies the document. For documents stored on disk this is a file URI. - // params.text the initial full content of the document. - this.connection.console.log(`${params.textDocument.uri} opened.`); - }); - this.connection.onDidChangeTextDocument((params) => { - // The content of a text document did change in VSCode. - // params.uri uniquely identifies the document. - // params.contentChanges describe the content changes to the document. - this.connection.console.log(`${params.textDocument.uri} changed: ${JSON.stringify(params.contentChanges)}`); - }); - this.connection.onDidCloseTextDocument((params) => { - // A text document got closed in VSCode. - // params.uri uniquely identifies the document. - this.connection.console.log(`${params.textDocument.uri} closed.`); - }); - */ - // listen for open, change and close text document events this.documents.listen(this.connection); @@ -196,7 +160,7 @@ export class LanguageServer { * Called when the client starts initialization */ @AddStackToErrorMessage - public onInitialize(params: InitializeParams) { + public onInitialize(params: InitializeParams): HandlerResult { let clientCapabilities = params.capabilities; // Does the client support the `workspace/configuration` request? @@ -244,36 +208,387 @@ export class LanguageServer { */ @AddStackToErrorMessage @TrackBusyStatus - private async onInitialized() { - let projectCreatedDeferred = new Deferred(); - this.initialProjectsCreated = projectCreatedDeferred.promise; + public async onInitialized() { + let projectCreatedDeferred = new Deferred(); + this.initialProjectsCreated = projectCreatedDeferred.promise; + + try { + if (this.hasConfigurationCapability) { + // Register for all configuration changes. + await this.connection.client.register( + DidChangeConfigurationNotification.type, + undefined + ); + } + + await this.syncProjects(); + + if (this.clientHasWorkspaceFolderCapability) { + this.connection.workspace.onDidChangeWorkspaceFolders(async (evt) => { + await this.syncProjects(); + }); + } + await this.waitAllProjectFirstRuns(false); + projectCreatedDeferred.resolve(); + } catch (e: any) { + this.sendCriticalFailure( + `Critical failure during BrighterScript language server startup. + Please file a github issue and include the contents of the 'BrighterScript Language Server' output channel. + + Error message: ${e.message}` + ); + throw e; + } + } + + /** + * Provide a list of completion items based on the current cursor position + */ + @AddStackToErrorMessage + @TrackBusyStatus + public async onCompletion(params: TextDocumentPositionParams) { + //ensure programs are initialized + await this.waitAllProjectFirstRuns(); + + let filePath = util.uriToPath(params.textDocument.uri); + + //wait until the file has settled + await this.onValidateSettled(); + + let completions = this + .getProjects() + .flatMap(workspace => workspace.builder.program.getCompletions(filePath, params.position)); + + for (let completion of completions) { + completion.commitCharacters = ['.']; + } + + return completions; + } + + /** + * Provide a full completion item from the selection + */ + @AddStackToErrorMessage + public onCompletionResolve(item: CompletionItem): CompletionItem { + if (item.data === 1) { + item.detail = 'TypeScript details'; + item.documentation = 'TypeScript documentation'; + } else if (item.data === 2) { + item.detail = 'JavaScript details'; + item.documentation = 'JavaScript documentation'; + } + return item; + } + + @AddStackToErrorMessage + @TrackBusyStatus + public async onCodeAction(params: CodeActionParams) { + //ensure programs are initialized + await this.waitAllProjectFirstRuns(); + + let srcPath = util.uriToPath(params.textDocument.uri); + + //wait until the file has settled + await this.onValidateSettled(); + + const codeActions = this + .getProjects() + //skip programs that don't have this file + .filter(x => x.builder?.program?.hasFile(srcPath)) + .flatMap(workspace => workspace.builder.program.getCodeActions(srcPath, params.range)); + + //clone the diagnostics for each code action, since certain diagnostics can have circular reference properties that kill the language server if serialized + for (const codeAction of codeActions) { + if (codeAction.diagnostics) { + codeAction.diagnostics = codeAction.diagnostics.map(x => util.toDiagnostic(x, params.textDocument.uri)); + } + } + return codeActions; + } + + + @AddStackToErrorMessage + private async onDidChangeConfiguration() { + if (this.hasConfigurationCapability) { + //if the user changes any config value, just mass-reload all projects + await this.reloadProjects(this.getProjects()); + // Reset all cached document settings + } else { + // this.globalSettings = ( + // (change.settings.languageServerExample || this.defaultSettings) + // ); + } + } + + /** + * Called when watched files changed (add/change/delete). + * The CLIENT is in charge of what files to watch, so all client + * implementations should ensure that all valid project + * file types are watched (.brs,.bs,.xml,manifest, and any json/text/image files) + */ + @AddStackToErrorMessage + @TrackBusyStatus + private async onDidChangeWatchedFiles(params: DidChangeWatchedFilesParams) { + //ensure programs are initialized + await this.waitAllProjectFirstRuns(); + + let projects = this.getProjects(); + + //convert all file paths to absolute paths + let changes = params.changes.map(x => { + return { + type: x.type, + srcPath: s`${URI.parse(x.uri).fsPath}` + }; + }); + + let keys = changes.map(x => x.srcPath); + + //filter the list of changes to only the ones that made it through the debounce unscathed + changes = changes.filter(x => keys.includes(x.srcPath)); + + //if we have changes to work with + if (changes.length > 0) { + + //if any bsconfig files were added or deleted, re-sync all projects instead of the more specific approach below + if (changes.find(x => (x.type === FileChangeType.Created || x.type === FileChangeType.Deleted) && path.basename(x.srcPath).toLowerCase() === 'bsconfig.json')) { + return this.syncProjects(); + } + + //reload any workspace whose bsconfig.json file has changed + { + let projectsToReload = [] as Project[]; + //get the file paths as a string array + let filePaths = changes.map((x) => x.srcPath); + + for (let project of projects) { + if (project.configFilePath && filePaths.includes(project.configFilePath)) { + projectsToReload.push(project); + } + } + if (projectsToReload.length > 0) { + //vsc can generate a ton of these changes, for vsc system files, so we need to bail if there's no work to do on any of our actual project files + //reload any projects that need to be reloaded + await this.reloadProjects(projectsToReload); + } + + //reassign `projects` to the non-reloaded projects + projects = projects.filter(x => !projectsToReload.includes(x)); + } + + //convert created folders into a list of files of their contents + const directoryChanges = changes + //get only creation items + .filter(change => change.type === FileChangeType.Created) + //keep only the directories + .filter(change => util.isDirectorySync(change.srcPath)); + + //remove the created directories from the changes array (we will add back each of their files next) + changes = changes.filter(x => !directoryChanges.includes(x)); + + //look up every file in each of the newly added directories + const newFileChanges = directoryChanges + //take just the path + .map(x => x.srcPath) + //exclude the roku deploy staging folder + .filter(dirPath => !dirPath.includes('.roku-deploy-staging')) + //get the files for each folder recursively + .flatMap(dirPath => { + //look up all files + let files = fastGlob.sync('**/*', { + absolute: true, + cwd: rokuDeployUtil.toForwardSlashes(dirPath) + }); + return files.map(x => { + return { + type: FileChangeType.Created, + srcPath: s`${x}` + }; + }); + }); + + //add the new file changes to the changes array. + changes.push(...newFileChanges as any); + + //give every workspace the chance to handle file changes + await Promise.all( + projects.map((project) => this.handleFileChanges(project, changes)) + ); + } + } + + @AddStackToErrorMessage + public async onHover(params: TextDocumentPositionParams) { + //ensure programs are initialized + await this.waitAllProjectFirstRuns(); + + const srcPath = util.uriToPath(params.textDocument.uri); + let projects = this.getProjects(); + let hovers = projects + //get hovers from all projects + .map((x) => x.builder.program.getHover(srcPath, params.position)) + //flatten to a single list + .flat(); + + const contents = [ + ...(hovers ?? []) + //pull all hover contents out into a flag array of strings + .map(x => { + return Array.isArray(x?.contents) ? x?.contents : [x?.contents]; + }).flat() + //remove nulls + .filter(x => !!x) + //dedupe hovers across all projects + .reduce((set, content) => set.add(content), new Set()).values() + ]; + + if (contents.length > 0) { + let hover: Hover = { + //use the range from the first hover + range: hovers[0]?.range, + //the contents of all hovers + contents: contents + }; + return hover; + } + } + + @AddStackToErrorMessage + @TrackBusyStatus + public async onWorkspaceSymbol(params: WorkspaceSymbolParams) { + await this.waitAllProjectFirstRuns(); + + const results = util.flatMap( + await Promise.all(this.getProjects().map(project => { + return project.builder.program.getWorkspaceSymbols(); + })), + c => c + ); + + // Remove duplicates + const allSymbols = Object.values(results.reduce((map, symbol) => { + const key = symbol.location.uri + symbol.name; + map[key] = symbol; + return map; + }, {})); + return allSymbols as SymbolInformation[]; + } + + @AddStackToErrorMessage + @TrackBusyStatus + public async onDocumentSymbol(params: DocumentSymbolParams) { + await this.waitAllProjectFirstRuns(); + + await this.keyedThrottler.onIdleOnce(util.uriToPath(params.textDocument.uri), true); + + const srcPath = util.uriToPath(params.textDocument.uri); + for (const project of this.getProjects()) { + const file = project.builder.program.getFile(srcPath); + if (isBrsFile(file)) { + return file.getDocumentSymbols(); + } + } + } + + @AddStackToErrorMessage + @TrackBusyStatus + public async onDefinition(params: TextDocumentPositionParams) { + await this.waitAllProjectFirstRuns(); + + const srcPath = util.uriToPath(params.textDocument.uri); + + const results = util.flatMap( + await Promise.all(this.getProjects().map(project => { + return project.builder.program.getDefinition(srcPath, params.position); + })), + c => c + ); + return results; + } + + @AddStackToErrorMessage + @TrackBusyStatus + public async onSignatureHelp(params: SignatureHelpParams) { + await this.waitAllProjectFirstRuns(); + + const filepath = util.uriToPath(params.textDocument.uri); + await this.keyedThrottler.onIdleOnce(filepath, true); + + try { + const signatures = util.flatMap( + await Promise.all(this.getProjects().map(project => project.builder.program.getSignatureHelp(filepath, params.position) + )), + c => c + ); + + const activeSignature = signatures.length > 0 ? 0 : null; + + const activeParameter = activeSignature >= 0 ? signatures[activeSignature]?.index : null; + + let results: SignatureHelp = { + signatures: signatures.map((s) => s.signature), + activeSignature: activeSignature, + activeParameter: activeParameter + }; + return results; + } catch (e: any) { + this.connection.console.error(`error in onSignatureHelp: ${e.stack ?? e.message ?? e}`); + return { + signatures: [], + activeSignature: 0, + activeParameter: 0 + }; + } + } + + @AddStackToErrorMessage + @TrackBusyStatus + public async onReferences(params: ReferenceParams) { + await this.waitAllProjectFirstRuns(); - try { - if (this.hasConfigurationCapability) { - // Register for all configuration changes. - await this.connection.client.register( - DidChangeConfigurationNotification.type, - undefined - ); - } + const position = params.position; + const srcPath = util.uriToPath(params.textDocument.uri); - await this.syncProjects(); + const results = util.flatMap( + await Promise.all(this.getProjects().map(project => { + return project.builder.program.getReferences(srcPath, position); + })), + c => c + ); + return results.filter((r) => r); + } - if (this.clientHasWorkspaceFolderCapability) { - this.connection.workspace.onDidChangeWorkspaceFolders(async (evt) => { - await this.syncProjects(); - }); + @AddStackToErrorMessage + @TrackBusyStatus + private async onFullSemanticTokens(params: SemanticTokensParams) { + await this.waitAllProjectFirstRuns(); + //wait for the file to settle (in case there are multiple file changes in quick succession) + await this.keyedThrottler.onIdleOnce(util.uriToPath(params.textDocument.uri), true); + //wait for the validation cycle to settle + await this.onValidateSettled(); + + const srcPath = util.uriToPath(params.textDocument.uri); + for (const project of this.projects) { + //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. + if (project.builder.program.hasFile(srcPath)) { + let semanticTokens = project.builder.program.getSemanticTokens(srcPath); + return { + data: encodeSemanticTokens(semanticTokens) + } as SemanticTokens; } - await this.waitAllProjectFirstRuns(false); - projectCreatedDeferred.resolve(); - } catch (e: any) { - this.sendCriticalFailure( - `Critical failure during BrighterScript language server startup. - Please file a github issue and include the contents of the 'BrighterScript Language Server' output channel. + } + } - Error message: ${e.message}` - ); - throw e; + @AddStackToErrorMessage + @TrackBusyStatus + public async onExecuteCommand(params: ExecuteCommandParams) { + await this.waitAllProjectFirstRuns(); + if (params.command === CustomCommands.TranspileFile) { + const result = await this.transpileFile(params.arguments[0]); + //back-compat: include `pathAbsolute` property so older vscode versions still work + (result as any).pathAbsolute = result.srcPath; + return result; } } @@ -492,71 +807,6 @@ export class LanguageServer { return newProject; } - /** - * Provide a list of completion items based on the current cursor position - */ - @AddStackToErrorMessage - @TrackBusyStatus - private async onCompletion(params: TextDocumentPositionParams) { - //ensure programs are initialized - await this.waitAllProjectFirstRuns(); - - let filePath = util.uriToPath(params.textDocument.uri); - - //wait until the file has settled - await this.onValidateSettled(); - - let completions = this - .getProjects() - .flatMap(workspace => workspace.builder.program.getCompletions(filePath, params.position)); - - for (let completion of completions) { - completion.commitCharacters = ['.']; - } - - return completions; - } - - /** - * Provide a full completion item from the selection - */ - @AddStackToErrorMessage - private onCompletionResolve(item: CompletionItem): CompletionItem { - if (item.data === 1) { - item.detail = 'TypeScript details'; - item.documentation = 'TypeScript documentation'; - } else if (item.data === 2) { - item.detail = 'JavaScript details'; - item.documentation = 'JavaScript documentation'; - } - return item; - } - - @AddStackToErrorMessage - @TrackBusyStatus - private async onCodeAction(params: CodeActionParams) { - //ensure programs are initialized - await this.waitAllProjectFirstRuns(); - - let srcPath = util.uriToPath(params.textDocument.uri); - - //wait until the file has settled - await this.onValidateSettled(); - - const codeActions = this - .getProjects() - //skip programs that don't have this file - .filter(x => x.builder?.program?.hasFile(srcPath)) - .flatMap(workspace => workspace.builder.program.getCodeActions(srcPath, params.range)); - - //clone the diagnostics for each code action, since certain diagnostics can have circular reference properties that kill the language server if serialized - for (const codeAction of codeActions) { - if (codeAction.diagnostics) { - codeAction.diagnostics = codeAction.diagnostics.map(x => util.toDiagnostic(x, params.textDocument.uri)); - } - } - return codeActions; - } /** * Reload each of the specified workspaces @@ -596,167 +846,56 @@ export class LanguageServer { private getRootDir(workspace: Project) { let options = workspace?.builder?.program?.options; return options?.rootDir ?? options?.cwd; - } - - /** - * Sometimes users will alter their bsconfig files array, and will include standalone files. - * If this is the case, those standalone workspaces should be removed because the file was - * included in an actual program now. - * - * Sometimes files that used to be included are now excluded, so those open files need to be re-processed as standalone - */ - private async synchronizeStandaloneProjects() { - - //remove standalone workspaces that are now included in projects - for (let standaloneFilePath in this.standaloneFileProjects) { - let standaloneProject = this.standaloneFileProjects[standaloneFilePath]; - for (let project of this.projects) { - await standaloneProject.firstRunPromise; - - let dest = rokuDeploy.getDestPath( - standaloneFilePath, - project?.builder?.program?.options?.files ?? [], - this.getRootDir(project) - ); - //destroy this standalone workspace because the file has now been included in an actual workspace, - //or if the workspace wants the file - if (project?.builder?.program?.hasFile(standaloneFilePath) || dest) { - standaloneProject.builder.dispose(); - delete this.standaloneFileProjects[standaloneFilePath]; - } - } - } - - //create standalone projects for open files that no longer have a project - let textDocuments = this.documents.all(); - outer: for (let textDocument of textDocuments) { - let filePath = URI.parse(textDocument.uri).fsPath; - for (let project of this.getProjects()) { - let dest = rokuDeploy.getDestPath( - filePath, - project?.builder?.program?.options?.files ?? [], - this.getRootDir(project) - ); - //if this project has the file, or it wants the file, do NOT make a standaloneProject for this file - if (project?.builder?.program?.hasFile(filePath) || dest) { - continue outer; - } - } - //if we got here, no workspace has this file, so make a standalone file workspace - let project = await this.createStandaloneFileProject(filePath); - await project.firstRunPromise; - } - } - - @AddStackToErrorMessage - private async onDidChangeConfiguration() { - if (this.hasConfigurationCapability) { - //if the user changes any config value, just mass-reload all projects - await this.reloadProjects(this.getProjects()); - // Reset all cached document settings - } else { - // this.globalSettings = ( - // (change.settings.languageServerExample || this.defaultSettings) - // ); - } - } - - /** - * Called when watched files changed (add/change/delete). - * The CLIENT is in charge of what files to watch, so all client - * implementations should ensure that all valid project - * file types are watched (.brs,.bs,.xml,manifest, and any json/text/image files) - */ - @AddStackToErrorMessage - @TrackBusyStatus - private async onDidChangeWatchedFiles(params: DidChangeWatchedFilesParams) { - //ensure programs are initialized - await this.waitAllProjectFirstRuns(); - - let projects = this.getProjects(); - - //convert all file paths to absolute paths - let changes = params.changes.map(x => { - return { - type: x.type, - srcPath: s`${URI.parse(x.uri).fsPath}` - }; - }); - - let keys = changes.map(x => x.srcPath); - - //filter the list of changes to only the ones that made it through the debounce unscathed - changes = changes.filter(x => keys.includes(x.srcPath)); - - //if we have changes to work with - if (changes.length > 0) { - - //if any bsconfig files were added or deleted, re-sync all projects instead of the more specific approach below - if (changes.find(x => (x.type === FileChangeType.Created || x.type === FileChangeType.Deleted) && path.basename(x.srcPath).toLowerCase() === 'bsconfig.json')) { - return this.syncProjects(); - } - - //reload any workspace whose bsconfig.json file has changed - { - let projectsToReload = [] as Project[]; - //get the file paths as a string array - let filePaths = changes.map((x) => x.srcPath); - - for (let project of projects) { - if (project.configFilePath && filePaths.includes(project.configFilePath)) { - projectsToReload.push(project); - } - } - if (projectsToReload.length > 0) { - //vsc can generate a ton of these changes, for vsc system files, so we need to bail if there's no work to do on any of our actual project files - //reload any projects that need to be reloaded - await this.reloadProjects(projectsToReload); - } - - //reassign `projects` to the non-reloaded projects - projects = projects.filter(x => !projectsToReload.includes(x)); - } - - //convert created folders into a list of files of their contents - const directoryChanges = changes - //get only creation items - .filter(change => change.type === FileChangeType.Created) - //keep only the directories - .filter(change => util.isDirectorySync(change.srcPath)); - - //remove the created directories from the changes array (we will add back each of their files next) - changes = changes.filter(x => !directoryChanges.includes(x)); + } - //look up every file in each of the newly added directories - const newFileChanges = directoryChanges - //take just the path - .map(x => x.srcPath) - //exclude the roku deploy staging folder - .filter(dirPath => !dirPath.includes('.roku-deploy-staging')) - //get the files for each folder recursively - .flatMap(dirPath => { - //look up all files - let files = fastGlob.sync('**/*', { - absolute: true, - cwd: rokuDeployUtil.toForwardSlashes(dirPath) - }); - return files.map(x => { - return { - type: FileChangeType.Created, - srcPath: s`${x}` - }; - }); - }); + /** + * Sometimes users will alter their bsconfig files array, and will include standalone files. + * If this is the case, those standalone workspaces should be removed because the file was + * included in an actual program now. + * + * Sometimes files that used to be included are now excluded, so those open files need to be re-processed as standalone + */ + private async synchronizeStandaloneProjects() { - //add the new file changes to the changes array. - changes.push(...newFileChanges as any); + //remove standalone workspaces that are now included in projects + for (let standaloneFilePath in this.standaloneFileProjects) { + let standaloneProject = this.standaloneFileProjects[standaloneFilePath]; + for (let project of this.projects) { + await standaloneProject.firstRunPromise; - //give every workspace the chance to handle file changes - await Promise.all( - projects.map((project) => this.handleFileChanges(project, changes)) - ); + let dest = rokuDeploy.getDestPath( + standaloneFilePath, + project?.builder?.program?.options?.files ?? [], + this.getRootDir(project) + ); + //destroy this standalone workspace because the file has now been included in an actual workspace, + //or if the workspace wants the file + if (project?.builder?.program?.hasFile(standaloneFilePath) || dest) { + standaloneProject.builder.dispose(); + delete this.standaloneFileProjects[standaloneFilePath]; + } + } } + //create standalone projects for open files that no longer have a project + let textDocuments = this.documents.all(); + outer: for (let textDocument of textDocuments) { + let filePath = URI.parse(textDocument.uri).fsPath; + for (let project of this.getProjects()) { + let dest = rokuDeploy.getDestPath( + filePath, + project?.builder?.program?.options?.files ?? [], + this.getRootDir(project) + ); + //if this project has the file, or it wants the file, do NOT make a standaloneProject for this file + if (project?.builder?.program?.hasFile(filePath) || dest) { + continue outer; + } + } + //if we got here, no workspace has this file, so make a standalone file workspace + let project = await this.createStandaloneFileProject(filePath); + await project.firstRunPromise; + } } /** @@ -837,42 +976,6 @@ export class LanguageServer { } } - @AddStackToErrorMessage - private async onHover(params: TextDocumentPositionParams) { - //ensure programs are initialized - await this.waitAllProjectFirstRuns(); - - const srcPath = util.uriToPath(params.textDocument.uri); - let projects = this.getProjects(); - let hovers = projects - //get hovers from all projects - .map((x) => x.builder.program.getHover(srcPath, params.position)) - //flatten to a single list - .flat(); - - const contents = [ - ...(hovers ?? []) - //pull all hover contents out into a flag array of strings - .map(x => { - return Array.isArray(x?.contents) ? x?.contents : [x?.contents]; - }).flat() - //remove nulls - .filter(x => !!x) - //dedupe hovers across all projects - .reduce((set, content) => set.add(content), new Set()).values() - ]; - - if (contents.length > 0) { - let hover: Hover = { - //use the range from the first hover - range: hovers[0]?.range, - //the contents of all hovers - contents: contents - }; - return hover; - } - } - @AddStackToErrorMessage private async onDocumentClose(event: TextDocumentChangeEvent): Promise { const { document } = event; @@ -943,111 +1046,6 @@ export class LanguageServer { } } - @AddStackToErrorMessage - @TrackBusyStatus - public async onWorkspaceSymbol(params: WorkspaceSymbolParams) { - await this.waitAllProjectFirstRuns(); - - const results = util.flatMap( - await Promise.all(this.getProjects().map(project => { - return project.builder.program.getWorkspaceSymbols(); - })), - c => c - ); - - // Remove duplicates - const allSymbols = Object.values(results.reduce((map, symbol) => { - const key = symbol.location.uri + symbol.name; - map[key] = symbol; - return map; - }, {})); - return allSymbols as SymbolInformation[]; - } - - @AddStackToErrorMessage - @TrackBusyStatus - public async onDocumentSymbol(params: DocumentSymbolParams) { - await this.waitAllProjectFirstRuns(); - - await this.keyedThrottler.onIdleOnce(util.uriToPath(params.textDocument.uri), true); - - const srcPath = util.uriToPath(params.textDocument.uri); - for (const project of this.getProjects()) { - const file = project.builder.program.getFile(srcPath); - if (isBrsFile(file)) { - return file.getDocumentSymbols(); - } - } - } - - @AddStackToErrorMessage - @TrackBusyStatus - private async onDefinition(params: TextDocumentPositionParams) { - await this.waitAllProjectFirstRuns(); - - const srcPath = util.uriToPath(params.textDocument.uri); - - const results = util.flatMap( - await Promise.all(this.getProjects().map(project => { - return project.builder.program.getDefinition(srcPath, params.position); - })), - c => c - ); - return results; - } - - @AddStackToErrorMessage - @TrackBusyStatus - private async onSignatureHelp(params: SignatureHelpParams) { - await this.waitAllProjectFirstRuns(); - - const filepath = util.uriToPath(params.textDocument.uri); - await this.keyedThrottler.onIdleOnce(filepath, true); - - try { - const signatures = util.flatMap( - await Promise.all(this.getProjects().map(project => project.builder.program.getSignatureHelp(filepath, params.position) - )), - c => c - ); - - const activeSignature = signatures.length > 0 ? 0 : null; - - const activeParameter = activeSignature >= 0 ? signatures[activeSignature]?.index : null; - - let results: SignatureHelp = { - signatures: signatures.map((s) => s.signature), - activeSignature: activeSignature, - activeParameter: activeParameter - }; - return results; - } catch (e: any) { - this.connection.console.error(`error in onSignatureHelp: ${e.stack ?? e.message ?? e}`); - return { - signatures: [], - activeSignature: 0, - activeParameter: 0 - }; - } - } - - @AddStackToErrorMessage - @TrackBusyStatus - private async onReferences(params: ReferenceParams) { - await this.waitAllProjectFirstRuns(); - - const position = params.position; - const srcPath = util.uriToPath(params.textDocument.uri); - - const results = util.flatMap( - await Promise.all(this.getProjects().map(project => { - return project.builder.program.getReferences(srcPath, position); - })), - c => c - ); - return results.filter((r) => r); - } - private onValidateSettled() { return Promise.all([ //wait for the validator to start running (or timeout if it never did) @@ -1057,27 +1055,6 @@ export class LanguageServer { ]); } - @AddStackToErrorMessage - @TrackBusyStatus - private async onFullSemanticTokens(params: SemanticTokensParams) { - await this.waitAllProjectFirstRuns(); - //wait for the file to settle (in case there are multiple file changes in quick succession) - await this.keyedThrottler.onIdleOnce(util.uriToPath(params.textDocument.uri), true); - //wait for the validation cycle to settle - await this.onValidateSettled(); - - const srcPath = util.uriToPath(params.textDocument.uri); - for (const project of this.projects) { - //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. - if (project.builder.program.hasFile(srcPath)) { - let semanticTokens = project.builder.program.getSemanticTokens(srcPath); - return { - data: encodeSemanticTokens(semanticTokens) - } as SemanticTokens; - } - } - } - private diagnosticCollection = new DiagnosticCollection(); private async sendDiagnostics() { @@ -1103,18 +1080,6 @@ export class LanguageServer { }); } - @AddStackToErrorMessage - @TrackBusyStatus - public async onExecuteCommand(params: ExecuteCommandParams) { - await this.waitAllProjectFirstRuns(); - if (params.command === CustomCommands.TranspileFile) { - const result = await this.transpileFile(params.arguments[0]); - //back-compat: include `pathAbsolute` property so older vscode versions still work - (result as any).pathAbsolute = result.srcPath; - return result; - } - } - private async transpileFile(srcPath: string) { //wait all program first runs await this.waitAllProjectFirstRuns(); @@ -1184,3 +1149,49 @@ function TrackBusyStatus(target: any, propertyKey: string, descriptor: PropertyD }, originalMethod.name); }; } + +type Methods = { + [K in keyof T]: T[K] extends Function ? K : never; +}[keyof T]; + +type AllMethods = { + [K in Methods]: T[K]; +}; + +type FilterMethodsStartsWith = { + [K in keyof T as K extends `${U}${string}` ? K : never]: T[K]; +}; + +type FirstArgumentType = T extends (...args: [infer U, ...any[]]) => any ? U : never; + +type FirstArgumentTypesOfFilteredMethods = { + [K in keyof FilterMethodsStartsWith]: FirstArgumentType[K]>; +}; + +// Example object +class Example { + onEvent1(arg1: number) { + // Some implementation + } + + handleEvent(arg1: string, arg2: boolean) { + // Some implementation + } + + onSomething(arg1: Date, arg2: number[]) { + // Some implementation + } + + notRelatedMethod() { + // Some implementation + } +} + +type Handler = { + [K in keyof T as K extends `on${string}` ? K : never]: + T[K] extends (arg: infer U) => void ? (arg: U) => void : never; +}; +// Extracts the argument type from the function and constructs the desired interface +type OnHandler = { + [K in keyof Handler]: Handler[K] extends (arg: infer U) => void ? U : never; +}; From fc2b7b2dc6c68e61744af96155c28299ba64a3c1 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 25 Dec 2023 21:58:55 -0500 Subject: [PATCH 009/119] Add semantic token support --- package-lock.json | 80 +++++------ package.json | 6 +- src/LanguageServer.spec.ts | 8 +- src/LanguageServer.ts | 150 +++++++------------- src/lsp/LspProject.ts | 20 ++- src/lsp/Project.ts | 59 ++++++-- src/lsp/ProjectManager.ts | 73 ++++++---- src/lsp/worker/MessageHandler.ts | 42 ++++-- src/lsp/worker/WorkerThreadProject.ts | 23 ++- src/lsp/worker/WorkerThreadProjectRunner.ts | 3 +- tsconfig.json | 1 + 11 files changed, 269 insertions(+), 196 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2f3f9cefd..9d81b3e39 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,9 +35,9 @@ "roku-deploy": "^3.11.2", "serialize-error": "^7.0.1", "source-map": "^0.7.4", - "vscode-languageserver": "7.0.0", - "vscode-languageserver-protocol": "3.16.0", - "vscode-languageserver-textdocument": "^1.0.1", + "vscode-languageserver": "^9.0.1", + "vscode-languageserver-protocol": "^3.17.5", + "vscode-languageserver-textdocument": "^1.0.11", "vscode-uri": "^2.1.1", "xml2js": "^0.5.0", "yargs": "^16.2.0" @@ -6810,42 +6810,42 @@ } }, "node_modules/vscode-languageserver": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-7.0.0.tgz", - "integrity": "sha512-60HTx5ID+fLRcgdHfmz0LDZAXYEV68fzwG0JWwEPBode9NuMYTIxuYXPg4ngO8i8+Ou0lM7y6GzaYWbiDL0drw==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-9.0.1.tgz", + "integrity": "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==", "dependencies": { - "vscode-languageserver-protocol": "3.16.0" + "vscode-languageserver-protocol": "3.17.5" }, "bin": { "installServerIntoExtension": "bin/installServerIntoExtension" } }, "node_modules/vscode-languageserver-protocol": { - "version": "3.16.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.16.0.tgz", - "integrity": "sha512-sdeUoAawceQdgIfTI+sdcwkiK2KU+2cbEYA0agzM2uqaUy2UpnnGHtWTHVEtS0ES4zHU0eMFRGN+oQgDxlD66A==", + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", "dependencies": { - "vscode-jsonrpc": "6.0.0", - "vscode-languageserver-types": "3.16.0" + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" } }, "node_modules/vscode-languageserver-protocol/node_modules/vscode-jsonrpc": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-6.0.0.tgz", - "integrity": "sha512-wnJA4BnEjOSyFMvjZdpiOwhSq9uDoK8e/kpRJDTaMYzwlkrhG1fwDIZI94CLsLzlCK5cIbMMtFlJlfR57Lavmg==", + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==", "engines": { - "node": ">=8.0.0 || >=10.0.0" + "node": ">=14.0.0" } }, "node_modules/vscode-languageserver-textdocument": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.1.tgz", - "integrity": "sha512-UIcJDjX7IFkck7cSkNNyzIz5FyvpQfY7sdzVy+wkKN/BLaD4DQ0ppXQrKePomCxTS7RrolK1I0pey0bG9eh8dA==" + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.11.tgz", + "integrity": "sha512-X+8T3GoiwTVlJbicx/sIAF+yuJAqz8VvwJyoMVhwEMoEKE/fkDmrqUgDMyBECcM2A2frVZIUj5HI/ErRXCfOeA==" }, "node_modules/vscode-languageserver-types": { - "version": "3.16.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0.tgz", - "integrity": "sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA==" + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==" }, "node_modules/vscode-uri": { "version": "2.1.2", @@ -12205,38 +12205,38 @@ "dev": true }, "vscode-languageserver": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-7.0.0.tgz", - "integrity": "sha512-60HTx5ID+fLRcgdHfmz0LDZAXYEV68fzwG0JWwEPBode9NuMYTIxuYXPg4ngO8i8+Ou0lM7y6GzaYWbiDL0drw==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-9.0.1.tgz", + "integrity": "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==", "requires": { - "vscode-languageserver-protocol": "3.16.0" + "vscode-languageserver-protocol": "3.17.5" } }, "vscode-languageserver-protocol": { - "version": "3.16.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.16.0.tgz", - "integrity": "sha512-sdeUoAawceQdgIfTI+sdcwkiK2KU+2cbEYA0agzM2uqaUy2UpnnGHtWTHVEtS0ES4zHU0eMFRGN+oQgDxlD66A==", + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", "requires": { - "vscode-jsonrpc": "6.0.0", - "vscode-languageserver-types": "3.16.0" + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" }, "dependencies": { "vscode-jsonrpc": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-6.0.0.tgz", - "integrity": "sha512-wnJA4BnEjOSyFMvjZdpiOwhSq9uDoK8e/kpRJDTaMYzwlkrhG1fwDIZI94CLsLzlCK5cIbMMtFlJlfR57Lavmg==" + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==" } } }, "vscode-languageserver-textdocument": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.1.tgz", - "integrity": "sha512-UIcJDjX7IFkck7cSkNNyzIz5FyvpQfY7sdzVy+wkKN/BLaD4DQ0ppXQrKePomCxTS7RrolK1I0pey0bG9eh8dA==" + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.11.tgz", + "integrity": "sha512-X+8T3GoiwTVlJbicx/sIAF+yuJAqz8VvwJyoMVhwEMoEKE/fkDmrqUgDMyBECcM2A2frVZIUj5HI/ErRXCfOeA==" }, "vscode-languageserver-types": { - "version": "3.16.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0.tgz", - "integrity": "sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA==" + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==" }, "vscode-uri": { "version": "2.1.2", diff --git a/package.json b/package.json index 746663ecf..9dac33c93 100644 --- a/package.json +++ b/package.json @@ -147,9 +147,9 @@ "roku-deploy": "^3.11.2", "serialize-error": "^7.0.1", "source-map": "^0.7.4", - "vscode-languageserver": "7.0.0", - "vscode-languageserver-protocol": "3.16.0", - "vscode-languageserver-textdocument": "^1.0.1", + "vscode-languageserver": "^9.0.1", + "vscode-languageserver-protocol": "^3.17.5", + "vscode-languageserver-textdocument": "^1.0.11", "vscode-uri": "^2.1.1", "xml2js": "^0.5.0", "yargs": "^16.2.0" diff --git a/src/LanguageServer.spec.ts b/src/LanguageServer.spec.ts index ce3f56cca..2c230598c 100644 --- a/src/LanguageServer.spec.ts +++ b/src/LanguageServer.spec.ts @@ -155,6 +155,10 @@ describe('LanguageServer', () => { }); }); + it('notifies user of critical runtime crash when activating a project', () => { + throw new Error('Implement me!'); + }); + describe('sendDiagnostics', () => { it('waits for program to finish loading before sending diagnostics', async () => { server.onInitialize({ @@ -1107,7 +1111,7 @@ describe('LanguageServer', () => { }); }); - it('semantic tokens request waits until after validation has finished', async () => { + it.only('semantic tokens request waits until after validation has finished', async () => { fsExtra.outputFileSync(s`${rootDir}/source/main.bs`, ` sub main() print \`hello world\` @@ -1133,7 +1137,7 @@ describe('LanguageServer', () => { fsExtra.outputFileSync(s`${rootDir}/source/sgnode.bs`, getContents()); server.run(); await server['syncProjects'](); - expectZeroDiagnostics(server['projectManager'].projects[0].builder.program); + expectZeroDiagnostics(server['projectManager'].get.projects[0].builder.program); fsExtra.outputFileSync(s`${rootDir}/source/sgnode.bs`, getContents()); const changeWatchedFilesPromise = server['onDidChangeWatchedFiles']({ diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 30a490234..5ae4295d6 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -25,7 +25,9 @@ import type { HandlerResult, InitializeError, InitializeResult, - InitializedParams + CompletionParams, + ResultProgressReporter, + WorkDoneProgressReporter } from 'vscode-languageserver/node'; import { SemanticTokensRequest, @@ -137,7 +139,7 @@ export class LanguageServer implements OnHandler { } } - //TODO switch to a more specific connection function call once they actually add it + //Register semantic token requestsTODO switch to a more specific connection function call once they actually add it this.connection.onRequest(SemanticTokensRequest.method, this.onFullSemanticTokens.bind(this)); // The content of a text document has changed. This event is emitted @@ -171,34 +173,34 @@ export class LanguageServer implements OnHandler { //return the capabilities of the server return { capabilities: { - textDocumentSync: TextDocumentSyncKind.Full, - // Tell the client that the server supports code completion - completionProvider: { - resolveProvider: true, - //anytime the user types a period, auto-show the completion results - triggerCharacters: ['.'], - allCommitCharacters: ['.', '@'] - }, - documentSymbolProvider: true, - workspaceSymbolProvider: true, + // textDocumentSync: TextDocumentSyncKind.Full, + // // Tell the client that the server supports code completion + // completionProvider: { + // resolveProvider: true, + // //anytime the user types a period, auto-show the completion results + // triggerCharacters: ['.'], + // allCommitCharacters: ['.', '@'] + // }, + // documentSymbolProvider: true, + // workspaceSymbolProvider: true, semanticTokensProvider: { legend: semanticTokensLegend, full: true } as SemanticTokensOptions, - referencesProvider: true, - codeActionProvider: { - codeActionKinds: [CodeActionKind.Refactor] - }, - signatureHelpProvider: { - triggerCharacters: ['(', ','] - }, - definitionProvider: true, - hoverProvider: true, - executeCommandProvider: { - commands: [ - CustomCommands.TranspileFile - ] - } + // referencesProvider: true, + // codeActionProvider: { + // codeActionKinds: [CodeActionKind.Refactor] + // }, + // signatureHelpProvider: { + // triggerCharacters: ['(', ','] + // }, + // definitionProvider: true, + // hoverProvider: true, + // executeCommandProvider: { + // commands: [ + // CustomCommands.TranspileFile + // ] + // } } as ServerCapabilities }; } @@ -209,9 +211,6 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage @TrackBusyStatus public async onInitialized() { - let projectCreatedDeferred = new Deferred(); - this.initialProjectsCreated = projectCreatedDeferred.promise; - try { if (this.hasConfigurationCapability) { // Register for all configuration changes. @@ -228,8 +227,6 @@ export class LanguageServer implements OnHandler { await this.syncProjects(); }); } - await this.waitAllProjectFirstRuns(false); - projectCreatedDeferred.resolve(); } catch (e: any) { this.sendCriticalFailure( `Critical failure during BrighterScript language server startup. @@ -246,22 +243,11 @@ export class LanguageServer implements OnHandler { */ @AddStackToErrorMessage @TrackBusyStatus - public async onCompletion(params: TextDocumentPositionParams) { - //ensure programs are initialized - await this.waitAllProjectFirstRuns(); - - let filePath = util.uriToPath(params.textDocument.uri); - - //wait until the file has settled - await this.onValidateSettled(); - - let completions = this - .getProjects() - .flatMap(workspace => workspace.builder.program.getCompletions(filePath, params.position)); - - for (let completion of completions) { - completion.commitCharacters = ['.']; - } + public async onCompletion1(params: CompletionParams, workDoneProgress: WorkDoneProgressReporter, resultProgress?: ResultProgressReporter) { + const completions = await this.projectManager.getCompletions( + util.uriToPath(params.textDocument.uri), + params.position + ); return completions; } @@ -559,25 +545,16 @@ export class LanguageServer implements OnHandler { return results.filter((r) => r); } + @AddStackToErrorMessage @TrackBusyStatus private async onFullSemanticTokens(params: SemanticTokensParams) { - await this.waitAllProjectFirstRuns(); - //wait for the file to settle (in case there are multiple file changes in quick succession) - await this.keyedThrottler.onIdleOnce(util.uriToPath(params.textDocument.uri), true); - //wait for the validation cycle to settle - await this.onValidateSettled(); - const srcPath = util.uriToPath(params.textDocument.uri); - for (const project of this.projects) { - //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. - if (project.builder.program.hasFile(srcPath)) { - let semanticTokens = project.builder.program.getSemanticTokens(srcPath); - return { - data: encodeSemanticTokens(semanticTokens) - } as SemanticTokens; - } - } + const result = await this.projectManager.getSemanticTokens(srcPath); + + return { + data: encodeSemanticTokens(result) + } as SemanticTokens; } @AddStackToErrorMessage @@ -617,9 +594,6 @@ export class LanguageServer implements OnHandler { } private busyStatusIndex = -1; - - private initialProjectsCreated: Promise; - /** * Ask the client for the list of `files.exclude` patterns. Useful when determining if we should process a file */ @@ -708,21 +682,7 @@ export class LanguageServer implements OnHandler { * Wait for all programs' first run to complete */ private async waitAllProjectFirstRuns(waitForFirstProject = true) { - if (waitForFirstProject) { - await this.initialProjectsCreated; - } - - for (let project of this.getProjects()) { - try { - await project.firstRunPromise; - } catch (e: any) { - status = 'critical-error'; - //the first run failed...that won't change unless we reload the workspace, so replace with resolved promise - //so we don't show this error again - project.firstRunPromise = Promise.resolve(); - this.sendCriticalFailure(`BrighterScript language server failed to start: \n${e.message}`); - } - } + //TODO delete me } /** @@ -1059,24 +1019,24 @@ export class LanguageServer implements OnHandler { private async sendDiagnostics() { await this.sendDiagnosticsThrottler.run(async () => { - // await this.projectManager.onSettle(); - //wait for all programs to finish running. This ensures the `Program` exists. - await Promise.all( - this.projects.map(x => x.firstRunPromise) - ); + // // await this.projectManager.onSettle(); + // //wait for all programs to finish running. This ensures the `Program` exists. + // await Promise.all( + // this.projects.map(x => x.firstRunPromise) + // ); - //Get only the changes to diagnostics since the last time we sent them to the client - const patch = this.diagnosticCollection.getPatch(this.projects); + // //Get only the changes to diagnostics since the last time we sent them to the client + // const patch = this.diagnosticCollection.getPatch(this.projects); - for (let filePath in patch) { - const uri = URI.file(filePath).toString(); - const diagnostics = patch[filePath].map(d => util.toDiagnostic(d, uri)); + // for (let filePath in patch) { + // const uri = URI.file(filePath).toString(); + // const diagnostics = patch[filePath].map(d => util.toDiagnostic(d, uri)); - this.connection.sendDiagnostics({ - uri: uri, - diagnostics: diagnostics - }); - } + // this.connection.sendDiagnostics({ + // uri: uri, + // diagnostics: diagnostics + // }); + // } }); } diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index ac5f71be8..82e2f43b5 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,4 +1,5 @@ -import type { Diagnostic } from 'vscode-languageserver-types'; +import type { CompletionItem, Diagnostic, Position } from 'vscode-languageserver'; +import { SemanticToken } from '..'; /** * Defines the contract between the ProjectManager and the main or worker thread Project classes @@ -19,12 +20,23 @@ export interface LspProject { * Initialize and start running the project. This will scan for all files, and build a full project in memory, then validate the project * @param options */ - activate(options: ActivateOptions): Promise; + activate(options: ActivateOptions): MaybePromise; /** * Get the list of all diagnostics from this project */ - getDiagnostics(): Promise; + getDiagnostics(): MaybePromise; + + /** + * Get the full list of semantic tokens for the given file path + * @param srcPath absolute path to the source file + */ + getSemanticTokens(srcPath: string): MaybePromise; + + /** + * Does this project have the specified filie + */ + hasFile(srcPath: string): MaybePromise; /** * Release all resources so this file can be safely garbage collected @@ -54,3 +66,5 @@ export interface ActivateOptions { export interface LspDiagnostic extends Diagnostic { uri: string; } + +type MaybePromise = T | Promise; diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index f40f4fb65..24eda43b2 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -6,15 +6,16 @@ import type { ActivateOptions, LspProject } from './LspProject'; import type { CompilerPlugin } from '../interfaces'; import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; +import { Deferred } from '../deferred'; export class Project implements LspProject { - /** * Activates this project. Every call to `activate` should completely reset the project, clear all used ram and start from scratch. * @param options */ public async activate(options: ActivateOptions) { this.dispose(); + this.isActivated = new Deferred(); this.projectPath = options.projectPath; this.workspaceFolder = options.workspaceFolder; @@ -25,7 +26,7 @@ export class Project implements LspProject { this.builder.logger.prefix = `[prj${this.projectNumber}]`; this.builder.logger.log(`Created project #${this.projectNumber} for: "${this.projectPath}"`); - let cwd; + let cwd: string; //if the config file exists, use it and its folder as cwd if (this.configFilePath && await util.pathExists(this.configFilePath)) { cwd = path.dirname(this.configFilePath); @@ -65,18 +66,49 @@ export class Project implements LspProject { }); this.emit('flush-diagnostics', { project: this }); } + this.isActivated.resolve(); } + /** + * Gets resolved when the project has finished activating + */ + private isActivated: Deferred; + public getDiagnostics() { - return Promise.resolve( - this.builder.getDiagnostics().map(x => { - const uri = URI.file(x.file.srcPath).toString(); - return { - ...util.toDiagnostic(x, uri), - uri: uri - }; - }) - ); + return this.builder.getDiagnostics().map(x => { + const uri = URI.file(x.file.srcPath).toString(); + return { + ...util.toDiagnostic(x, uri), + uri: uri + }; + }); + } + + /** + * Promise that resolves the next time the system is idle. If the system is already idle, it will resolve immediately + */ + private async onIdle(): Promise { + await Promise.all([ + this.isActivated.promise + ]); + } + + /** + * Determine if this project has the specified file + * @param srcPath the absolute path to the file + * @returns true if the project has the file, false if it does not + */ + public hasFile(srcPath: string) { + return this.builder.program.hasFile(srcPath); + } + + /** + * Get the full list of semantic tokens for the given file path + * @param srcPath absolute path to the source file + */ + public async getSemanticTokens(srcPath: string) { + await this.onIdle(); + return this.builder.program.getSemanticTokens(srcPath); } /** @@ -168,5 +200,10 @@ export class Project implements LspProject { public dispose() { this.builder?.dispose(); this.emitter.removeAllListeners(); + if (this.isActivated?.isCompleted === false) { + this.isActivated.reject( + new Error('Project was disposed, activation has been aborted') + ); + } } } diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 38ef35c9a..2cbc7b6e7 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -5,41 +5,17 @@ import * as EventEmitter from 'eventemitter3'; import type { LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; +import type { Position } from 'vscode-languageserver'; /** * Manages all brighterscript projects for the language server */ export class ProjectManager { - /** * Collection of all projects */ public projects: LspProject[] = []; - /** - * A unique project counter to help distinguish log entries in lsp mode - */ - private static projectNumberSequence = 0; - - - public on(eventName: 'critical-failure', handler: (data: { project: LspProject; message: string }) => void); - public on(eventName: 'flush-diagnostics', handler: (data: { project: LspProject }) => void); - public on(eventName: string, handler: (payload: any) => void) { - this.emitter.on(eventName, handler); - return () => { - this.emitter.removeListener(eventName, handler); - }; - } - - private emit(eventName: 'critical-failure', data: { project: LspProject; message: string }); - private emit(eventName: 'flush-diagnostics', data: { project: LspProject }); - private async emit(eventName: string, data?) { - //emit these events on next tick, otherwise they will be processed immediately which could cause issues - await util.sleep(0); - this.emitter.emit(eventName, data); - } - private emitter = new EventEmitter(); - /** * Given a list of all desired projects, create any missing projects and destroy and projects that are no longer available * Treat workspaces that don't have a bsconfig.json as a project. @@ -88,6 +64,28 @@ export class ProjectManager { ); } + public async getSemanticTokens(srcPath: string) { + for (const project of this.projects) { + //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. + if (await project.hasFile(srcPath)) { + const result = await Promise.resolve( + project.getSemanticTokens(srcPath) + ); + return result; + } + } + } + + public async getCompletions(srcPath: string, position: Position) { + const completions = await Promise.all( + this.projects.map(x => x.getCompletions(srcPath, position)) + ); + + for (let completion of completions) { + completion.commitCharacters = ['.']; + } + } + /** * Scan a given workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. * If none are found, then the workspaceFolder itself is treated as a project @@ -164,6 +162,11 @@ export class ProjectManager { project?.dispose(); } + /** + * A unique project counter to help distinguish log entries in lsp mode + */ + private static projectNumberSequence = 0; + /** * Create a project for the given config * @param config @@ -188,7 +191,27 @@ export class ProjectManager { workspaceFolder: config1.workspaceFolder, projectNumber: config1.projectNumber ?? ProjectManager.projectNumberSequence++ }); + console.log('Activated'); } + + public on(eventName: 'critical-failure', handler: (data: { project: LspProject; message: string }) => void); + public on(eventName: 'flush-diagnostics', handler: (data: { project: LspProject }) => void); + public on(eventName: string, handler: (payload: any) => void) { + this.emitter.on(eventName, handler); + return () => { + this.emitter.removeListener(eventName, handler); + }; + } + + private emit(eventName: 'critical-failure', data: { project: LspProject; message: string }); + private emit(eventName: 'flush-diagnostics', data: { project: LspProject }); + private async emit(eventName: string, data?) { + //emit these events on next tick, otherwise they will be processed immediately which could cause issues + await util.sleep(0); + this.emitter.emit(eventName, data); + } + private emitter = new EventEmitter(); + } export interface WorkspaceConfig { diff --git a/src/lsp/worker/MessageHandler.ts b/src/lsp/worker/MessageHandler.ts index 39ba6471a..cb4edd25e 100644 --- a/src/lsp/worker/MessageHandler.ts +++ b/src/lsp/worker/MessageHandler.ts @@ -8,14 +8,14 @@ interface PseudoMessagePort { postMessage: typeof parentPort['postMessage']; } -export class MessageHandler { +export class MessageHandler> { constructor( options: { name?: string; port: PseudoMessagePort; - onRequest?: (message: WorkerMessage) => any; - onResponse?: (message: WorkerMessage) => any; - onUpdate?: (message: WorkerMessage) => any; + onRequest?: (message: WorkerRequest) => any; + onResponse?: (message: WorkerResponse) => any; + onUpdate?: (message: WorkerUpdate) => any; } ) { this.name = options?.name; @@ -76,17 +76,17 @@ export class MessageHandler { * @param name the name of the request * @param options the request options */ - public async sendRequest(name: string, options?: { data: any[]; id?: number }) { + public async sendRequest(name: TRequestName, options?: { data: any[]; id?: number }) { const request: WorkerMessage = { type: 'request', - name: name, + name: name as string, data: options?.data ?? [], id: options?.id ?? this.idSequence++ }; const responsePromise = this.onResponse(request.id); this.port.postMessage(request); const response = await responsePromise; - if (response.error) { + if ('error' in response) { const error = this.objectToError(response.error); (error as any)._response = response; //throw the error so it causes a rejected promise (like we'd expect) @@ -98,7 +98,7 @@ export class MessageHandler { /** * Send a request to the worker, and wait for a response. * @param request the request we are responding to - * @param response the data to be sent as the response + * @param options options for this request */ public sendResponse(request: WorkerMessage, options?: { data: any } | { error: Error } | undefined) { const response: WorkerResponse = { @@ -162,15 +162,33 @@ export class MessageHandler { } } -export interface WorkerMessage { +export interface WorkerRequest { id: number; - type: 'request' | 'response' | 'update'; + type: 'request'; name: string; - data?: T; + data?: TData; } -export interface WorkerResponse extends WorkerMessage { + +export interface WorkerResponse { + id: number; + type: 'response'; + name: string; + data?: TData; /** * An error occurred on the remote side. There will be no `.data` value */ error?: Error; } + +export interface WorkerUpdate { + id: number; + type: 'update'; + name: string; + data?: TData; +} + +export type WorkerMessage = WorkerRequest | WorkerResponse | WorkerUpdate; + +type MethodNames = { + [K in keyof T]: T[K] extends (...args: any[]) => any ? K : never; +}[keyof T]; diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 93ad5f32f..9951e814c 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -9,6 +9,7 @@ import { isMainThread, parentPort } from 'worker_threads'; import { WorkerThreadProjectRunner } from './WorkerThreadProjectRunner'; import type { Project } from '../Project'; import { WorkerPool } from './WorkerPool'; +import { SemanticToken } from '../..'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -40,7 +41,7 @@ export class WorkerThreadProject implements LspProject { // start a new worker thread or get an unused existing thread this.worker = workerPool.getWorker(); - this.messageHandler = new MessageHandler({ + this.messageHandler = new MessageHandler({ name: 'MainThread', port: this.worker, onRequest: this.processRequest.bind(this), @@ -50,14 +51,28 @@ export class WorkerThreadProject implements LspProject { } public async getDiagnostics() { - const diagnostics = await this.messageHandler.sendRequest('getDiagnostics'); - return diagnostics.data; + const response = await this.messageHandler.sendRequest('getDiagnostics'); + return response.data; + } + + public async hasFile(srcPath: string) { + const response = await this.messageHandler.sendRequest('hasFile'); + return response.data; + } + + /** + * Get the full list of semantic tokens for the given file path + * @param srcPath absolute path to the source file + */ + public async getSemanticTokens(srcPath: string) { + const response = await this.messageHandler.sendRequest('getSemanticTokens'); + return response.data; } /** * Handles request/response/update messages from the worker thread */ - private messageHandler: MessageHandler; + private messageHandler: MessageHandler; private processRequest(request: WorkerMessage) { diff --git a/src/lsp/worker/WorkerThreadProjectRunner.ts b/src/lsp/worker/WorkerThreadProjectRunner.ts index f9b2858d9..57b359c12 100644 --- a/src/lsp/worker/WorkerThreadProjectRunner.ts +++ b/src/lsp/worker/WorkerThreadProjectRunner.ts @@ -1,3 +1,4 @@ +import type { LspProject } from '../LspProject'; import { Project } from '../Project'; import type { WorkerMessage } from './MessageHandler'; import { MessageHandler } from './MessageHandler'; @@ -10,7 +11,7 @@ export class WorkerThreadProjectRunner { public run(parentPort: MessagePort) { //make a new instance of the project (which is the same way we run it in the main thread). const project = new Project(); - const messageHandler = new MessageHandler({ + const messageHandler = new MessageHandler({ name: 'WorkerThread', port: parentPort, onRequest: async (request: WorkerMessage) => { diff --git a/tsconfig.json b/tsconfig.json index 92b871a8c..bb7ad8dbe 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,6 +13,7 @@ "experimentalDecorators": true, "preserveConstEnums": true, "downlevelIteration": true, + "noEmitOnError": false, "noUnusedLocals": true, "allowSyntheticDefaultImports": true, "resolveJsonModule": true, From 47bc3dec413daedada7555d903de905b12c46a24 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 2 Jan 2024 07:55:34 -0500 Subject: [PATCH 010/119] Flush diagnostics on a per-project basis --- package-lock.json | 45 ++-- package.json | 3 +- src/DiagnosticCollection.spec.ts | 264 +++++++++++++++----- src/DiagnosticCollection.ts | 107 ++++++-- src/LanguageServer.spec.ts | 4 +- src/LanguageServer.ts | 96 +++---- src/lsp/LspProject.ts | 13 +- src/lsp/Project.spec.ts | 10 +- src/lsp/Project.ts | 27 +- src/lsp/ProjectManager.spec.ts | 10 +- src/lsp/ProjectManager.ts | 31 +-- src/lsp/worker/MessageHandler.spec.ts | 9 +- src/lsp/worker/MessageHandler.ts | 2 +- src/lsp/worker/WorkerPool.spec.ts | 2 +- src/lsp/worker/WorkerThreadProject.spec.ts | 2 +- src/lsp/worker/WorkerThreadProject.ts | 11 +- src/lsp/worker/WorkerThreadProjectRunner.ts | 6 + src/util.ts | 2 +- 18 files changed, 427 insertions(+), 217 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9d81b3e39..3350f7e74 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49,6 +49,7 @@ "@guyplusplus/turndown-plugin-gfm": "^1.0.7", "@types/benchmark": "^1.0.31", "@types/chai": "^4.1.2", + "@types/clone": "^2.1.4", "@types/command-line-args": "^5.0.0", "@types/command-line-usage": "^5.0.1", "@types/debounce-promise": "^3.1.1", @@ -666,6 +667,12 @@ "integrity": "sha512-o3SGYRlOpvLFpwJA6Sl1UPOwKFEvE4FxTEB/c9XHI2whdnd4kmPVkNLL8gY4vWGBxWWDumzLbKsAhEH5SKn37Q==", "dev": true }, + "node_modules/@types/clone": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/clone/-/clone-2.1.4.tgz", + "integrity": "sha512-NKRWaEGaVGVLnGLB2GazvDaZnyweW9FJLLFL5LhywGJB3aqGMT9R/EUoJoSRP4nzofYnZysuDmrEJtJdAqUOtQ==", + "dev": true + }, "node_modules/@types/command-line-args": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.0.0.tgz", @@ -1787,15 +1794,6 @@ "wrap-ansi": "^7.0.0" } }, - "node_modules/clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=", - "dev": true, - "engines": { - "node": ">=0.8" - } - }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -2124,6 +2122,15 @@ "clone": "^1.0.2" } }, + "node_modules/defaults/node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "dev": true, + "engines": { + "node": ">=0.8" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -7609,6 +7616,12 @@ "integrity": "sha512-o3SGYRlOpvLFpwJA6Sl1UPOwKFEvE4FxTEB/c9XHI2whdnd4kmPVkNLL8gY4vWGBxWWDumzLbKsAhEH5SKn37Q==", "dev": true }, + "@types/clone": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@types/clone/-/clone-2.1.4.tgz", + "integrity": "sha512-NKRWaEGaVGVLnGLB2GazvDaZnyweW9FJLLFL5LhywGJB3aqGMT9R/EUoJoSRP4nzofYnZysuDmrEJtJdAqUOtQ==", + "dev": true + }, "@types/command-line-args": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.0.0.tgz", @@ -8391,12 +8404,6 @@ "wrap-ansi": "^7.0.0" } }, - "clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=", - "dev": true - }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -8658,6 +8665,14 @@ "dev": true, "requires": { "clone": "^1.0.2" + }, + "dependencies": { + "clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "dev": true + } } }, "delayed-stream": { diff --git a/package.json b/package.json index 9dac33c93..cc0bd1e7c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "brighterscript", - "version": "0.65.14", + "version": "0.65.14-local", "description": "A superset of Roku's BrightScript language.", "scripts": { "preversion": "npm run build && npm run lint && npm run test", @@ -76,6 +76,7 @@ "@guyplusplus/turndown-plugin-gfm": "^1.0.7", "@types/benchmark": "^1.0.31", "@types/chai": "^4.1.2", + "@types/clone": "^2.1.4", "@types/command-line-args": "^5.0.0", "@types/command-line-usage": "^5.0.1", "@types/debounce-promise": "^3.1.1", diff --git a/src/DiagnosticCollection.spec.ts b/src/DiagnosticCollection.spec.ts index 2a4d09c1f..1eb228f29 100644 --- a/src/DiagnosticCollection.spec.ts +++ b/src/DiagnosticCollection.spec.ts @@ -1,122 +1,260 @@ -import type { BsDiagnostic } from '.'; import { DiagnosticCollection } from './DiagnosticCollection'; -import type { Project } from './LanguageServer'; -import type { ProgramBuilder } from './ProgramBuilder'; -import type { BscFile } from './interfaces'; import util from './util'; import { expect } from './chai-config.spec'; +import { Project } from './lsp/Project'; +import type { LspDiagnostic, LspProject } from './lsp/LspProject'; +import { URI } from 'vscode-uri'; +import { rootDir } from './testHelpers.spec'; +import * as path from 'path'; +import { standardizePath } from './util'; +import { interpolatedRange } from './astUtils/creators'; -describe('DiagnosticCollection', () => { +describe.only('DiagnosticCollection', () => { let collection: DiagnosticCollection; - let diagnostics: BsDiagnostic[]; - let projects: Project[]; + let project: Project; + beforeEach(() => { collection = new DiagnosticCollection(); - diagnostics = []; - //make simple mock of workspace to pass tests - projects = [{ - firstRunPromise: Promise.resolve(), - builder: { - getDiagnostics: () => diagnostics - } as ProgramBuilder - }] as Project[]; + project = new Project(); + project.projectPath = rootDir; }); - function testPatch(expected: Record) { - const patch = collection.getPatch(projects); + function testPatch(options: { + project?: LspProject; + diagnosticsByFile?: Record>; + expected?: Record; + }) { + + const patch = collection.getPatch(options.project ?? project, createDiagnostics(options.diagnosticsByFile ?? {})); //convert the patch into our test structure const actual = {}; - for (const filePath in patch) { + for (let filePath in patch) { + filePath = path.resolve(rootDir, filePath); actual[filePath] = patch[filePath].map(x => x.message); } + //sanitize expected paths + let expected = {}; + for (let key in options.expected ?? {}) { + const srcPath = standardizePath( + path.resolve(rootDir, key) + ); + expected[srcPath] = options.expected[key]; + } expect(actual).to.eql(expected); } + it('computes patch for specific project', () => { + const project1 = new Project(); + const project2 = new Project(); + //should be all diagnostics from project1 + testPatch({ + project: project1, + diagnosticsByFile: { + 'alpha.brs': ['a1', 'a2'], + 'beta.brs': ['b1', 'b2'] + }, + expected: { + 'alpha.brs': ['a1', 'a2'], + 'beta.brs': ['b1', 'b2'] + } + }); + + //set project2 diagnostics that overlap a little with project1 + testPatch({ + project: project2, + diagnosticsByFile: { + 'beta.brs': ['b2', 'b3'], + 'charlie.brs': ['c1', 'c2'] + }, + //the patch should only include new diagnostics + expected: { + 'beta.brs': ['b1', 'b2', 'b3'], + 'charlie.brs': ['c1', 'c2'] + } + }); + + //set project 1 diagnostics again (same diagnostics) + testPatch({ + project: project1, + diagnosticsByFile: { + 'alpha.brs': ['a1', 'a2'], + 'beta.brs': ['b1', 'b2'] + }, + //patch should be empty because nothing changed + expected: { + } + }); + }); + it('does not crash for diagnostics with missing locations', () => { - const [d1] = addDiagnostics('file1.brs', ['I have no location']); - delete d1.range; + const d1: LspDiagnostic = { + code: 123, + range: undefined, + uri: undefined, + message: 'I have no location' + }; testPatch({ - 'file1.brs': ['I have no location'] + diagnosticsByFile: { + 'source/file1.brs': [d1 as any] + }, + expected: { + 'source/file1.brs': ['I have no location'] + } }); }); it('returns full list of diagnostics on first call, and nothing on second call', () => { - addDiagnostics('file1.brs', ['message1', 'message2']); - addDiagnostics('file2.brs', ['message3', 'message4']); //first patch should return all testPatch({ - 'file1.brs': ['message1', 'message2'], - 'file2.brs': ['message3', 'message4'] + diagnosticsByFile: { + 'file1.brs': ['message1', 'message2'], + 'file2.brs': ['message3', 'message4'] + }, + expected: { + 'file1.brs': ['message1', 'message2'], + 'file2.brs': ['message3', 'message4'] + } }); //second patch should return empty (because nothing has changed) - testPatch({}); + testPatch({ + diagnosticsByFile: { + 'file1.brs': ['message1', 'message2'], + 'file2.brs': ['message3', 'message4'] + }, + expected: { + } + }); }); it('removes diagnostics in patch', () => { - addDiagnostics('file1.brs', ['message1', 'message2']); - addDiagnostics('file2.brs', ['message3', 'message4']); //first patch should return all testPatch({ - 'file1.brs': ['message1', 'message2'], - 'file2.brs': ['message3', 'message4'] + diagnosticsByFile: { + 'file1.brs': ['message1', 'message2'], + 'file2.brs': ['message3', 'message4'] + }, + expected: { + 'file1.brs': ['message1', 'message2'], + 'file2.brs': ['message3', 'message4'] + } }); - removeDiagnostic('file1.brs', 'message1'); - removeDiagnostic('file1.brs', 'message2'); + + //removing the diagnostics should result in a new patch with those diagnostics removed testPatch({ - 'file1.brs': [] + diagnosticsByFile: { + 'file1.brs': [], + 'file2.brs': ['message3', 'message4'] + }, + expected: { + 'file1.brs': [] + } }); }); it('adds diagnostics in patch', () => { - addDiagnostics('file1.brs', ['message1', 'message2']); testPatch({ - 'file1.brs': ['message1', 'message2'] + diagnosticsByFile: { + 'file1.brs': ['message1', 'message2'] + }, + expected: { + 'file1.brs': ['message1', 'message2'] + } }); - addDiagnostics('file2.brs', ['message3', 'message4']); testPatch({ - 'file2.brs': ['message3', 'message4'] + diagnosticsByFile: { + 'file1.brs': ['message1', 'message2'], + 'file2.brs': ['message3', 'message4'] + }, + expected: { + 'file2.brs': ['message3', 'message4'] + } }); }); it('sends full list when file diagnostics have changed', () => { - addDiagnostics('file1.brs', ['message1', 'message2']); testPatch({ - 'file1.brs': ['message1', 'message2'] + diagnosticsByFile: { + 'file1.brs': ['message1', 'message2'] + }, + expected: { + 'file1.brs': ['message1', 'message2'] + } }); - addDiagnostics('file1.brs', ['message3', 'message4']); testPatch({ - 'file1.brs': ['message1', 'message2', 'message3', 'message4'] + diagnosticsByFile: { + 'file1.brs': ['message1', 'message2', 'message3', 'message4'] + }, + expected: { + 'file1.brs': ['message1', 'message2', 'message3', 'message4'] + } }); }); - function removeDiagnostic(srcPath: string, message: string) { - for (let i = 0; i < diagnostics.length; i++) { - const diagnostic = diagnostics[i]; - if (diagnostic.file.srcPath === srcPath && diagnostic.message === message) { - diagnostics.splice(i, 1); - return; + it('handles when diagnostics.projects is already defined and already includes this project', () => { + testPatch({ + diagnosticsByFile: { + 'file1.brs': [{ + message: 'message1', + range: interpolatedRange, + uri: undefined, + projects: [project] + } as any] + }, + expected: { + 'file1.brs': ['message1'] } - } - throw new Error(`Cannot find diagnostic ${srcPath}:${message}`); - } + }); + }); - function addDiagnostics(srcPath: string, messages: string[]) { - const newDiagnostics = []; - for (const message of messages) { - newDiagnostics.push({ - file: { - srcPath: srcPath - } as BscFile, - range: util.createRange(0, 0, 0, 0), - //the code doesn't matter as long as the messages are different, so just enforce unique messages for this test files - code: 123, - message: message + describe('getRemovedPatch', () => { + it('returns empty array for file that was removed', () => { + collection['previousDiagnosticsByFile'] = { + [`lib1.brs`]: [] + }; + expect( + collection['getRemovedPatch']({ + [`lib2.brs`]: [] + }) + ).to.eql({ + [`lib1.brs`]: [] }); + }); + }); + + describe('diagnosticListsAreIdentical', () => { + it('returns false for different diagnostics in same-sized list', () => { + expect( + collection['diagnosticListsAreIdentical']([ + { key: 'one' } as any + ], [ + { key: 'two' } as any + ]) + ).to.be.false; + }); + }); + + function createDiagnostics(diagnosticsByFile: Record>) { + const newDiagnostics: LspDiagnostic[] = []; + for (let [srcPath, diagnostics] of Object.entries(diagnosticsByFile)) { + srcPath = path.resolve(rootDir, srcPath); + for (const d of diagnostics) { + let diagnostic = d as LspDiagnostic; + if (typeof d === 'string') { + diagnostic = { + uri: undefined, + range: util.createRange(0, 0, 0, 0), + //the code doesn't matter as long as the messages are different, so just enforce unique messages for this test files + code: 123, + message: d + }; + } + diagnostic.uri = URI.file(srcPath).toString(); + newDiagnostics.push(diagnostic); + } } - diagnostics.push(...newDiagnostics); - return newDiagnostics as typeof diagnostics; + return newDiagnostics; } }); diff --git a/src/DiagnosticCollection.ts b/src/DiagnosticCollection.ts index 98d067024..a38d8376e 100644 --- a/src/DiagnosticCollection.ts +++ b/src/DiagnosticCollection.ts @@ -1,12 +1,20 @@ -import type { BsDiagnostic } from './interfaces'; -import type { Project } from './LanguageServer'; +import { URI } from 'vscode-uri'; +import type { LspDiagnostic, LspProject } from './lsp/LspProject'; import { util } from './util'; +import { firstBy } from 'thenby'; export class DiagnosticCollection { - private previousDiagnosticsByFile = {} as Record; + private previousDiagnosticsByFile: Record = {}; - public getPatch(projects: Project[]) { - const diagnosticsByFile = this.getDiagnosticsByFileFromProjects(projects); + /** + * Get a patch of any changed diagnostics since last time. This takes a single project and diagnostics, but evaulates + * the patch based on all previously seen projects. It's supposed to be a rolling patch. + * This will include _ALL_ diagnostics for a file if any diagnostics have changed for that file, due to how the language server expects diagnostics to be sent. + * @param projects + * @returns + */ + public getPatch(project: LspProject, diagnostics: LspDiagnostic[]) { + const diagnosticsByFile = this.getDiagnosticsByFile(project, diagnostics as KeyedDiagnostic[]); const patch = { ...this.getRemovedPatch(diagnosticsByFile), @@ -19,26 +27,49 @@ export class DiagnosticCollection { return patch; } - private getDiagnosticsByFileFromProjects(projects: Project[]) { - const result = {} as Record; - - //get all diagnostics for all projects - let diagnostics = Array.prototype.concat.apply([] as KeyedDiagnostic[], - projects.map((x) => x.builder.getDiagnostics()) - ) as KeyedDiagnostic[]; + /** + * Get all the previous diagnostics, remove any that were exclusive to the current project, then mix in the project's new diagnostics. + * @param project the latest project that should have its diagnostics refreshed + * @param thisProjectDiagnostics diagnostics for the project + * @returns + */ + private getDiagnosticsByFile(project: LspProject, thisProjectDiagnostics: KeyedDiagnostic[]) { + const result = this.clonePreviousDiagnosticsByFile(); + + const diagnosticsByKey = new Map(); + + //delete all diagnostics linked to this project + for (const srcPath in result) { + const diagnostics = result[srcPath]; + for (let i = diagnostics.length - 1; i >= 0; i--) { + const diagnostic = diagnostics[i]; + + //remember this diagnostic key for use when deduping down below + diagnosticsByKey.set(diagnostic.key, diagnostic); + + const idx = diagnostic.projects.indexOf(project); + //unlink the diagnostic from this project + if (idx > -1) { + diagnostic.projects.splice(idx, 1); + } + //delete this diagnostic if it's no longer linked to any projects + if (diagnostic.projects.length === 0) { + diagnostics.splice(i, 1); + diagnosticsByKey.delete(diagnostic.key); + } + } + } - const keys = {}; //build the full current set of diagnostics by file - for (let diagnostic of diagnostics) { - const srcPath = diagnostic.file.srcPath ?? diagnostic.file.srcPath; + for (let diagnostic of thisProjectDiagnostics) { + const srcPath = URI.parse(diagnostic.uri).fsPath; //ensure the file entry exists if (!result[srcPath]) { result[srcPath] = []; } - const diagnosticMap = result[srcPath]; //fall back to a default range if missing - const range = diagnostic?.range ?? util.createRange(0, 0, 0, 0); + const range = diagnostic.range ?? util.createRange(0, 0, 0, 0); diagnostic.key = srcPath.toLowerCase() + '-' + @@ -49,15 +80,48 @@ export class DiagnosticCollection { range.end.character + diagnostic.message; + diagnostic.projects ??= [project]; + //don't include duplicates - if (!keys[diagnostic.key]) { - keys[diagnostic.key] = true; - diagnosticMap.push(diagnostic); + if (!diagnosticsByKey.has(diagnostic.key)) { + diagnosticsByKey.set(diagnostic.key, diagnostic); + + const diagnosticsForFile = result[srcPath]; + diagnosticsForFile.push(diagnostic); + } + + const projects = diagnosticsByKey.get(diagnostic.key).projects; + //link this project to the diagnostic + if (!projects.includes(project)) { + projects.push(project); } } + + //sort the list so it's easier to compare later + for (let key in result) { + result[key].sort(firstBy(x => x.key)); + } return result; } + /** + * Clone the previousDiagnosticsByFile, retaining the array of project references on each diagnostic + */ + private clonePreviousDiagnosticsByFile() { + let clone: typeof this.previousDiagnosticsByFile = {}; + for (let key in this.previousDiagnosticsByFile) { + clone[key] = []; + for (const diagnostic of this.previousDiagnosticsByFile[key]) { + clone[key].push({ + ...diagnostic, + //make a copy of the projects array (but keep the project references intact) + projects: [...diagnostic.projects] + }); + } + } + return clone; + } + /** * Get a patch for all the files that have been removed since last time */ @@ -117,6 +181,7 @@ export class DiagnosticCollection { } } -interface KeyedDiagnostic extends BsDiagnostic { +interface KeyedDiagnostic extends LspDiagnostic { key: string; + projects: LspProject[]; } diff --git a/src/LanguageServer.spec.ts b/src/LanguageServer.spec.ts index 2c230598c..05e21f492 100644 --- a/src/LanguageServer.spec.ts +++ b/src/LanguageServer.spec.ts @@ -4,7 +4,7 @@ import * as path from 'path'; import type { DidChangeWatchedFilesParams, Location } from 'vscode-languageserver'; import { FileChangeType, Range } from 'vscode-languageserver'; import { Deferred } from './deferred'; -import type { Project } from './LanguageServer'; +import type { Project } from './lsp/Project'; import { CustomCommands, LanguageServer } from './LanguageServer'; import type { SinonStub } from 'sinon'; import { createSandbox } from 'sinon'; @@ -1111,7 +1111,7 @@ describe('LanguageServer', () => { }); }); - it.only('semantic tokens request waits until after validation has finished', async () => { + it('semantic tokens request waits until after validation has finished', async () => { fsExtra.outputFileSync(s`${rootDir}/source/main.bs`, ` sub main() print \`hello world\` diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 5ae4295d6..3ddfa9dd1 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -42,7 +42,6 @@ import { import { URI } from 'vscode-uri'; import { TextDocument } from 'vscode-languageserver-textdocument'; import type { BsConfig } from './BsConfig'; -import { Deferred } from './deferred'; import { ProgramBuilder } from './ProgramBuilder'; import { standardizePath as s, util } from './util'; import { Logger } from './Logger'; @@ -54,6 +53,8 @@ import type { BusyStatus } from './BusyStatusTracker'; import { BusyStatusTracker } from './BusyStatusTracker'; import type { WorkspaceConfig } from './lsp/ProjectManager'; import { ProjectManager } from './lsp/ProjectManager'; +import type { LspDiagnostic, LspProject } from './lsp/LspProject'; +import type { Project } from './lsp/Project'; export class LanguageServer implements OnHandler { @@ -98,8 +99,6 @@ export class LanguageServer implements OnHandler { public validateThrottler = new Throttler(0); - private sendDiagnosticsThrottler = new Throttler(0); - private boundValidateAll = this.validateAll.bind(this); private validateAllThrottled() { @@ -111,10 +110,12 @@ export class LanguageServer implements OnHandler { //run the server public run() { this.projectManager = new ProjectManager(); - //anytime a project finishes validation, send diagnostics - this.projectManager.on('flush-diagnostics', () => { - void this.sendDiagnostics(); + + //anytime a project emits a collection of diagnostics, send them to the client + this.projectManager.on('diagnostics', (event) => { + void this.sendDiagnostics(event); }); + //allow the lsp to provide file contents //TODO handle this... // this.projectManager.addFileResolver(this.documentFileResolver.bind(this)); @@ -186,7 +187,7 @@ export class LanguageServer implements OnHandler { semanticTokensProvider: { legend: semanticTokensLegend, full: true - } as SemanticTokensOptions, + } as SemanticTokensOptions // referencesProvider: true, // codeActionProvider: { // codeActionKinds: [CodeActionKind.Refactor] @@ -642,9 +643,6 @@ export class LanguageServer implements OnHandler { ); await this.projectManager.syncProjects(workspaces); - - //flush diagnostics - await this.sendDiagnostics(); } /** @@ -953,6 +951,7 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage @TrackBusyStatus private async validateTextDocument(event: TextDocumentChangeEvent): Promise { + return; const { document } = event; //ensure programs are initialized await this.waitAllProjectFirstRuns(); @@ -1015,30 +1014,23 @@ export class LanguageServer implements OnHandler { ]); } - private diagnosticCollection = new DiagnosticCollection(); - - private async sendDiagnostics() { - await this.sendDiagnosticsThrottler.run(async () => { - // // await this.projectManager.onSettle(); - // //wait for all programs to finish running. This ensures the `Program` exists. - // await Promise.all( - // this.projects.map(x => x.firstRunPromise) - // ); - - // //Get only the changes to diagnostics since the last time we sent them to the client - // const patch = this.diagnosticCollection.getPatch(this.projects); + /** + * Send diagnostics to the client + */ + private async sendDiagnostics(options: { project: LspProject; diagnostics: LspDiagnostic[] }) { + const patch = this.diagnosticCollection.getPatch(options.project, options.diagnostics); - // for (let filePath in patch) { - // const uri = URI.file(filePath).toString(); - // const diagnostics = patch[filePath].map(d => util.toDiagnostic(d, uri)); + await Promise.all(Object.keys(patch).map(async (srcPath) => { + const uri = URI.file(srcPath).toString(); + const diagnostics = patch[srcPath].map(d => util.toDiagnostic(d, uri)); - // this.connection.sendDiagnostics({ - // uri: uri, - // diagnostics: diagnostics - // }); - // } - }); + await this.connection.sendDiagnostics({ + uri: uri, + diagnostics: diagnostics + }); + })); } + private diagnosticCollection = new DiagnosticCollection(); private async transpileFile(srcPath: string) { //wait all program first runs @@ -1051,6 +1043,11 @@ export class LanguageServer implements OnHandler { } } + private getProjects() { + //TODO delete this because projectManager handles all this stuff now + return []; + } + public dispose() { this.loggerSubscription?.(); this.validateThrottler.dispose(); @@ -1110,43 +1107,6 @@ function TrackBusyStatus(target: any, propertyKey: string, descriptor: PropertyD }; } -type Methods = { - [K in keyof T]: T[K] extends Function ? K : never; -}[keyof T]; - -type AllMethods = { - [K in Methods]: T[K]; -}; - -type FilterMethodsStartsWith = { - [K in keyof T as K extends `${U}${string}` ? K : never]: T[K]; -}; - -type FirstArgumentType = T extends (...args: [infer U, ...any[]]) => any ? U : never; - -type FirstArgumentTypesOfFilteredMethods = { - [K in keyof FilterMethodsStartsWith]: FirstArgumentType[K]>; -}; - -// Example object -class Example { - onEvent1(arg1: number) { - // Some implementation - } - - handleEvent(arg1: string, arg2: boolean) { - // Some implementation - } - - onSomething(arg1: Date, arg2: number[]) { - // Some implementation - } - - notRelatedMethod() { - // Some implementation - } -} - type Handler = { [K in keyof T as K extends `on${string}` ? K : never]: T[K] extends (arg: infer U) => void ? (arg: U) => void : never; diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 82e2f43b5..7b4b09739 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,5 +1,5 @@ -import type { CompletionItem, Diagnostic, Position } from 'vscode-languageserver'; -import { SemanticToken } from '..'; +import type { Diagnostic } from 'vscode-languageserver'; +import type { SemanticToken } from '../interfaces'; /** * Defines the contract between the ProjectManager and the main or worker thread Project classes @@ -38,6 +38,13 @@ export interface LspProject { */ hasFile(srcPath: string): MaybePromise; + /** + * An event that is emitted anytime the diagnostics for the project have changed (typically after a validate cycle has finished) + * @param eventName + * @param handler + */ + on(eventName: 'diagnostics', handler: (data: { diagnostics: LspDiagnostic[] }) => void); + /** * Release all resources so this file can be safely garbage collected */ @@ -67,4 +74,4 @@ export interface LspDiagnostic extends Diagnostic { uri: string; } -type MaybePromise = T | Promise; +export type MaybePromise = T | Promise; diff --git a/src/lsp/Project.spec.ts b/src/lsp/Project.spec.ts index 13401cb94..8e0dde431 100644 --- a/src/lsp/Project.spec.ts +++ b/src/lsp/Project.spec.ts @@ -8,7 +8,7 @@ import { Project } from './Project'; import { createSandbox } from 'sinon'; const sinon = createSandbox(); -describe.only('Project', () => { +describe('Project', () => { let project: Project; beforeEach(() => { @@ -26,16 +26,16 @@ describe.only('Project', () => { describe('on', () => { it('emits events', async () => { const stub = sinon.stub(); - const off = project.on('flush-diagnostics', stub); - await project['emit']('flush-diagnostics', { project: project }); + const off = project.on('diagnostics', stub); + await project['emit']('diagnostics', { project: project, diagnostics: [] }); expect(stub.callCount).to.eql(1); - await project['emit']('flush-diagnostics', { project: project }); + await project['emit']('diagnostics', { project: project, diagnostics: [] }); expect(stub.callCount).to.eql(2); off(); - await project['emit']('flush-diagnostics', { project: project }); + await project['emit']('diagnostics', { project: project, diagnostics: [] }); expect(stub.callCount).to.eql(2); }); }); diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 24eda43b2..39b372fd8 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -2,7 +2,7 @@ import { ProgramBuilder } from '../ProgramBuilder'; import * as EventEmitter from 'eventemitter3'; import util, { standardizePath as s } from '../util'; import * as path from 'path'; -import type { ActivateOptions, LspProject } from './LspProject'; +import type { ActivateOptions, LspDiagnostic, LspProject, MaybePromise } from './LspProject'; import type { CompilerPlugin } from '../interfaces'; import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; @@ -40,7 +40,10 @@ export class Project implements LspProject { this.builder.plugins.add({ name: 'bsc-language-server', afterProgramValidate: () => { - this.emit('flush-diagnostics', { project: this }); + this.emit('diagnostics', { + project: this, + diagnostics: this.getDiagnostics() + }); } } as CompilerPlugin); @@ -64,8 +67,11 @@ export class Project implements LspProject { ...DiagnosticMessages.brsConfigJsonIsDeprecated(), range: util.createRange(0, 0, 0, 0) }); - this.emit('flush-diagnostics', { project: this }); } + + //flush any diagnostics generated by this initial run + this.emit('diagnostics', { project: this, diagnostics: this.getDiagnostics() }); + this.isActivated.resolve(); } @@ -179,21 +185,24 @@ export class Project implements LspProject { return undefined; } - public on(eventName: 'critical-failure', handler: (data: { project: Project; message: string }) => void); - public on(eventName: 'flush-diagnostics', handler: (data: { project: Project }) => void); - public on(eventName: string, handler: (payload: any) => void) { - this.emitter.on(eventName, handler); + public on(eventName: 'critical-failure', handler: (data: { project: Project; message: string }) => MaybePromise); + public on(eventName: 'diagnostics', handler: (data: { diagnostics: LspDiagnostic[] }) => MaybePromise); + public on(eventName: 'all', handler: (eventName: string, data: any) => MaybePromise); + public on(eventName: string, handler: (...args: any[]) => MaybePromise) { + this.emitter.on(eventName, handler as any); return () => { - this.emitter.removeListener(eventName, handler); + this.emitter.removeListener(eventName, handler as any); }; } private emit(eventName: 'critical-failure', data: { project: Project; message: string }); - private emit(eventName: 'flush-diagnostics', data: { project: Project }); + private emit(eventName: 'diagnostics', data: { project: Project; diagnostics: LspDiagnostic[] }); private async emit(eventName: string, data?) { //emit these events on next tick, otherwise they will be processed immediately which could cause issues await util.sleep(0); this.emitter.emit(eventName, data); + //emit the 'all' event + this.emitter.emit('all', eventName, data); } private emitter = new EventEmitter(); diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index d095a64b8..eda6fb90d 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -9,7 +9,7 @@ import { WorkerThreadProject } from './worker/WorkerThreadProject'; import { wakeWorkerThreadPromise } from './worker/WorkerThreadProject.spec'; const sinon = createSandbox(); -describe.only('ProjectManager', () => { +describe('ProjectManager', () => { let manager: ProjectManager; before(async function workerThreadWarmup() { @@ -32,16 +32,16 @@ describe.only('ProjectManager', () => { describe('on', () => { it('emits events', async () => { const stub = sinon.stub(); - const off = manager.on('flush-diagnostics', stub); - await manager['emit']('flush-diagnostics', { project: undefined }); + const off = manager.on('diagnostics', stub); + await manager['emit']('diagnostics', { project: undefined, diagnostics: [] }); expect(stub.callCount).to.eql(1); - await manager['emit']('flush-diagnostics', { project: undefined }); + await manager['emit']('diagnostics', { project: undefined, diagnostics: [] }); expect(stub.callCount).to.eql(2); off(); - await manager['emit']('flush-diagnostics', { project: undefined }); + await manager['emit']('diagnostics', { project: undefined, diagnostics: [] }); expect(stub.callCount).to.eql(2); }); }); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 2cbc7b6e7..9b1cf4b79 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -2,7 +2,7 @@ import { standardizePath as s, util } from '../util'; import { rokuDeploy } from 'roku-deploy'; import * as path from 'path'; import * as EventEmitter from 'eventemitter3'; -import type { LspProject } from './LspProject'; +import type { LspDiagnostic, LspProject, MaybePromise } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; import type { Position } from 'vscode-languageserver'; @@ -77,13 +77,13 @@ export class ProjectManager { } public async getCompletions(srcPath: string, position: Position) { - const completions = await Promise.all( - this.projects.map(x => x.getCompletions(srcPath, position)) - ); + // const completions = await Promise.all( + // this.projects.map(x => x.getCompletions(srcPath, position)) + // ); - for (let completion of completions) { - completion.commitCharacters = ['.']; - } + // for (let completion of completions) { + // completion.commitCharacters = ['.']; + // } } /** @@ -184,6 +184,10 @@ export class ProjectManager { this.projects.push(project); + project.on('diagnostics', (event) => { + this.emit('diagnostics', { ...event, project: project }); + }); + //TODO subscribe to various events for this project await project.activate({ @@ -194,24 +198,23 @@ export class ProjectManager { console.log('Activated'); } - public on(eventName: 'critical-failure', handler: (data: { project: LspProject; message: string }) => void); - public on(eventName: 'flush-diagnostics', handler: (data: { project: LspProject }) => void); - public on(eventName: string, handler: (payload: any) => void) { - this.emitter.on(eventName, handler); + public on(eventName: 'critical-failure', handler: (data: { project: LspProject; message: string }) => MaybePromise); + public on(eventName: 'diagnostics', handler: (data: { project: LspProject; diagnostics: LspDiagnostic[] }) => MaybePromise); + public on(eventName: string, handler: (payload: any) => MaybePromise) { + this.emitter.on(eventName, handler as any); return () => { - this.emitter.removeListener(eventName, handler); + this.emitter.removeListener(eventName, handler as any); }; } private emit(eventName: 'critical-failure', data: { project: LspProject; message: string }); - private emit(eventName: 'flush-diagnostics', data: { project: LspProject }); + private emit(eventName: 'diagnostics', data: { project: LspProject; diagnostics: LspDiagnostic[] }); private async emit(eventName: string, data?) { //emit these events on next tick, otherwise they will be processed immediately which could cause issues await util.sleep(0); this.emitter.emit(eventName, data); } private emitter = new EventEmitter(); - } export interface WorkspaceConfig { diff --git a/src/lsp/worker/MessageHandler.spec.ts b/src/lsp/worker/MessageHandler.spec.ts index 7dcbfbc89..8f1937e5d 100644 --- a/src/lsp/worker/MessageHandler.spec.ts +++ b/src/lsp/worker/MessageHandler.spec.ts @@ -1,10 +1,11 @@ import { MessageChannel } from 'worker_threads'; import { MessageHandler } from './MessageHandler'; import { expect } from '../../chai-config.spec'; +import type { LspProject } from '../LspProject'; describe('MessageHandler', () => { - let server: MessageHandler; - let client: MessageHandler; + let server: MessageHandler; + let client: MessageHandler; let channel: MessageChannel; beforeEach(() => { @@ -27,10 +28,10 @@ describe('MessageHandler', () => { }); } }); - let client = new MessageHandler({ port: channel.port2 }); + let client = new MessageHandler({ port: channel.port2 }); let error: Error; try { - await client.sendRequest('doSomething'); + await client.sendRequest('activate'); } catch (e) { error = e as any; } diff --git a/src/lsp/worker/MessageHandler.ts b/src/lsp/worker/MessageHandler.ts index cb4edd25e..052413a20 100644 --- a/src/lsp/worker/MessageHandler.ts +++ b/src/lsp/worker/MessageHandler.ts @@ -79,7 +79,7 @@ export class MessageHandler> { public async sendRequest(name: TRequestName, options?: { data: any[]; id?: number }) { const request: WorkerMessage = { type: 'request', - name: name as string, + name: name as any, data: options?.data ?? [], id: options?.id ?? this.idSequence++ }; diff --git a/src/lsp/worker/WorkerPool.spec.ts b/src/lsp/worker/WorkerPool.spec.ts index 1365fac30..38815ac2f 100644 --- a/src/lsp/worker/WorkerPool.spec.ts +++ b/src/lsp/worker/WorkerPool.spec.ts @@ -2,7 +2,7 @@ import { expect } from 'chai'; import { WorkerPool } from './WorkerPool'; import type { Worker } from 'worker_threads'; -describe.only('WorkerPool', () => { +describe('WorkerPool', () => { let pool: WorkerPool; let workers: Worker[] = [] as any; diff --git a/src/lsp/worker/WorkerThreadProject.spec.ts b/src/lsp/worker/WorkerThreadProject.spec.ts index d4c2fe37c..8a5e660f6 100644 --- a/src/lsp/worker/WorkerThreadProject.spec.ts +++ b/src/lsp/worker/WorkerThreadProject.spec.ts @@ -23,7 +23,7 @@ after(() => { workerPool.dispose(); }); -describe.only('WorkerThreadProject', () => { +describe('WorkerThreadProject', () => { let project: WorkerThreadProject; before(async function workerThreadWarmup() { this.timeout(20_000); diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 9951e814c..3be386fe5 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -3,13 +3,13 @@ import { Worker } from 'worker_threads'; import type { WorkerMessage } from './MessageHandler'; import { MessageHandler } from './MessageHandler'; import util from '../../util'; -import type { LspDiagnostic } from '../LspProject'; +import type { LspDiagnostic, MaybePromise } from '../LspProject'; import { type ActivateOptions, type LspProject } from '../LspProject'; import { isMainThread, parentPort } from 'worker_threads'; import { WorkerThreadProjectRunner } from './WorkerThreadProjectRunner'; import type { Project } from '../Project'; import { WorkerPool } from './WorkerPool'; -import { SemanticToken } from '../..'; +import type { SemanticToken } from '../../interfaces'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -79,7 +79,8 @@ export class WorkerThreadProject implements LspProject { } private processUpdate(update: WorkerMessage) { - + //for now, all updates are treated like "events" + this.emit(update.name as any, update.data); } /** @@ -109,6 +110,7 @@ export class WorkerThreadProject implements LspProject { public configFilePath?: string; public on(eventName: 'critical-failure', handler: (data: { project: Project; message: string }) => void); + public on(eventName: 'diagnostics', handler: (data: { diagnostics: LspDiagnostic[] }) => MaybePromise); public on(eventName: string, handler: (payload: any) => void) { this.emitter.on(eventName, handler); return () => { @@ -117,10 +119,13 @@ export class WorkerThreadProject implements LspProject { } private emit(eventName: 'critical-failure', data: { project: Project; message: string }); + private emit(eventName: 'diagnostics', data: { diagnostics: LspDiagnostic[] }); private async emit(eventName: string, data?) { //emit these events on next tick, otherwise they will be processed immediately which could cause issues await util.sleep(0); this.emitter.emit(eventName, data); + //emit the 'all' event + this.emitter.emit('all', eventName, data); } private emitter = new EventEmitter(); diff --git a/src/lsp/worker/WorkerThreadProjectRunner.ts b/src/lsp/worker/WorkerThreadProjectRunner.ts index 57b359c12..d77a73dfc 100644 --- a/src/lsp/worker/WorkerThreadProjectRunner.ts +++ b/src/lsp/worker/WorkerThreadProjectRunner.ts @@ -30,5 +30,11 @@ export class WorkerThreadProjectRunner { } }); + + project.on('all', (eventName: string, data: any) => { + messageHandler.sendUpdate(eventName, { + data: data + }); + }); } } diff --git a/src/util.ts b/src/util.ts index 69fc1ff5d..5637fcfbf 100644 --- a/src/util.ts +++ b/src/util.ts @@ -4,7 +4,7 @@ import type { ParseError } from 'jsonc-parser'; import { parse as parseJsonc, printParseErrorCode } from 'jsonc-parser'; import * as path from 'path'; import { rokuDeploy, DefaultFiles, standardizePath as rokuDeployStandardizePath } from 'roku-deploy'; -import type { Diagnostic, Position, Range, Location, Disposable } from 'vscode-languageserver'; +import type { Diagnostic, Position, Range, Location } from 'vscode-languageserver'; import { URI } from 'vscode-uri'; import * as xml2js from 'xml2js'; import type { BsConfig } from './BsConfig'; From 7295d48c13882b58473bf394703d0a1b5e611d07 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 2 Jan 2024 15:10:13 -0500 Subject: [PATCH 011/119] Only spin up worker when tests require it --- src/DiagnosticCollection.spec.ts | 2 +- src/lsp/ProjectManager.spec.ts | 16 +++++++++------- src/lsp/worker/WorkerThreadProject.spec.ts | 10 ++++++++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/DiagnosticCollection.spec.ts b/src/DiagnosticCollection.spec.ts index 1eb228f29..a5ec31df5 100644 --- a/src/DiagnosticCollection.spec.ts +++ b/src/DiagnosticCollection.spec.ts @@ -9,7 +9,7 @@ import * as path from 'path'; import { standardizePath } from './util'; import { interpolatedRange } from './astUtils/creators'; -describe.only('DiagnosticCollection', () => { +describe('DiagnosticCollection', () => { let collection: DiagnosticCollection; let project: Project; diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index eda6fb90d..706214e26 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -2,21 +2,16 @@ import { expect } from 'chai'; import { ProjectManager } from './ProjectManager'; import { tempDir, rootDir } from '../testHelpers.spec'; import * as fsExtra from 'fs-extra'; -import { standardizePath as s } from '../util'; +import util, { standardizePath as s } from '../util'; import { createSandbox } from 'sinon'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import { wakeWorkerThreadPromise } from './worker/WorkerThreadProject.spec'; +import { getWakeWorkerThreadPromise } from './worker/WorkerThreadProject.spec'; const sinon = createSandbox(); describe('ProjectManager', () => { let manager: ProjectManager; - before(async function workerThreadWarmup() { - this.timeout(20_000); - await wakeWorkerThreadPromise; - }); - beforeEach(() => { manager = new ProjectManager(); fsExtra.emptyDirSync(tempDir); @@ -159,6 +154,13 @@ describe('ProjectManager', () => { s`${rootDir}/subdir2` ]); }); + }); + + describe('threading', () => { + before(async function workerThreadWarmup() { + this.timeout(20_000); + await getWakeWorkerThreadPromise(); + }); it('spawns a worker thread when threading is enabled', async () => { await manager.syncProjects([{ diff --git a/src/lsp/worker/WorkerThreadProject.spec.ts b/src/lsp/worker/WorkerThreadProject.spec.ts index 8a5e660f6..c2a7841da 100644 --- a/src/lsp/worker/WorkerThreadProject.spec.ts +++ b/src/lsp/worker/WorkerThreadProject.spec.ts @@ -17,7 +17,13 @@ export async function wakeWorkerThread() { } } -export const wakeWorkerThreadPromise = wakeWorkerThread(); +let wakeWorkerThreadPromise1: Promise; +export function getWakeWorkerThreadPromise() { + if (wakeWorkerThreadPromise1 === undefined) { + wakeWorkerThreadPromise1 = wakeWorkerThread(); + } + return wakeWorkerThreadPromise1; +} after(() => { workerPool.dispose(); @@ -27,7 +33,7 @@ describe('WorkerThreadProject', () => { let project: WorkerThreadProject; before(async function workerThreadWarmup() { this.timeout(20_000); - await wakeWorkerThreadPromise; + await getWakeWorkerThreadPromise(); }); beforeEach(() => { From 939d722c35e4cd0327f8ce09efa97b2e51f6c8a6 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 2 Jan 2024 15:10:30 -0500 Subject: [PATCH 012/119] semantic tokens race for first project --- src/deferred.ts | 12 ++++++ src/lsp/ProjectManager.spec.ts | 74 ++++++++++++++++++++++++++++++++++ src/lsp/ProjectManager.ts | 47 +++++++++++++++++---- 3 files changed, 125 insertions(+), 8 deletions(-) diff --git a/src/deferred.ts b/src/deferred.ts index 1adefe88c..749a09e4c 100644 --- a/src/deferred.ts +++ b/src/deferred.ts @@ -47,6 +47,12 @@ export class Deferred { } private _resolve: (value: T) => void; + public tryResolve(value?: T) { + if (!this.isCompleted) { + this.resolve(value); + } + } + /** * Reject the promise */ @@ -59,4 +65,10 @@ export class Deferred { this._reject(value); } private _reject: (value: TReject) => void; + + public tryReject(reason?: TReject) { + if (!this.isCompleted) { + this.reject(reason); + } + } } diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index 706214e26..9922ef43c 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -242,4 +242,78 @@ describe('ProjectManager', () => { } as any); }); }); + + describe('getSemanticTokens', () => { + it('waits until the project is ready', () => { + + }); + }); + + describe.only('raceUntil', () => { + beforeEach(() => { + }); + + async function doTest(expectedIndex: number, ...values: any[]) { + manager.projects = [{ index: 0 }, { index: 1 }, { index: 2 }] as any; + + let idx = 0; + expect( + await manager['findFirstMatchingProject']((project) => { + return values[idx++]; + }) + ).to.equal(manager.projects[expectedIndex]); + } + + async function sleepResolve(timeout: number, value: boolean) { + await util.sleep(timeout); + return value; + } + + + async function sleepReject(timeout: number, reason: string) { + await util.sleep(timeout); + throw new Error(reason); + } + + it('resolves sync values', async () => { + //return the first true value encountered. These are sync, so should resolve immediately + await doTest(0, true, false, false); + await doTest(1, false, true, false); + await doTest(2, false, false, true); + }); + + it('resolves async values', async () => { + //return the first true value encountered + await doTest(0, Promise.resolve(true), Promise.resolve(false), Promise.resolve(false)); + await doTest(1, Promise.resolve(false), Promise.resolve(true), Promise.resolve(false)); + await doTest(2, Promise.resolve(false), Promise.resolve(false), Promise.resolve(true)); + }); + + it('resolves async values in proper timing order', async () => { + //return the first true value encountered + await doTest(0, sleepResolve(0, true), sleepResolve(10, false), sleepResolve(20, false)); + await doTest(1, sleepResolve(0, false), sleepResolve(10, true), sleepResolve(20, false)); + await doTest(2, sleepResolve(0, false), sleepResolve(10, false), sleepResolve(20, true)); + }); + + it('resolves async values in proper timing order when all are true', async () => { + //return the first true value encountered + await doTest(0, sleepResolve(0, true), sleepResolve(10, true), sleepResolve(20, true)); + await doTest(1, sleepResolve(20, true), sleepResolve(0, true), sleepResolve(10, true)); + await doTest(2, sleepResolve(10, true), sleepResolve(20, true), sleepResolve(0, true)); + }); + + it('fails gracefully when an error occurs', async () => { + //return the first true value encountered + await doTest(0, sleepResolve(10, true), sleepReject(0, 'crash'), sleepResolve(20, true)); + await doTest(1, sleepResolve(20, true), sleepResolve(10, true), sleepReject(0, 'crash')); + await doTest(2, sleepReject(0, 'crash'), sleepResolve(20, true), sleepResolve(10, true)); + }); + + it('returns undefined when all promises return false', async () => { + await doTest(undefined, false, false, false); + await doTest(undefined, sleepResolve(0, false), sleepResolve(10, false), sleepResolve(20, false)); + await doTest(undefined, sleepReject(0, 'crash'), sleepReject(10, 'crash'), sleepReject(20, 'crash')); + }); + }); }); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 9b1cf4b79..a72083b79 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -6,6 +6,7 @@ import type { LspDiagnostic, LspProject, MaybePromise } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; import type { Position } from 'vscode-languageserver'; +import { Deferred } from '../deferred'; /** * Manages all brighterscript projects for the language server @@ -64,15 +65,45 @@ export class ProjectManager { ); } - public async getSemanticTokens(srcPath: string) { - for (const project of this.projects) { - //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. - if (await project.hasFile(srcPath)) { - const result = await Promise.resolve( - project.getSemanticTokens(srcPath) - ); - return result; + /** + * Return the first project where the async matcher returns true + * @param callback + * @returns + */ + private findFirstMatchingProject(callback: (project: LspProject) => boolean | PromiseLike) { + const deferred = new Deferred(); + let projectCount = this.projects.length; + let doneCount = 0; + this.projects.map(async (project) => { + try { + if (await Promise.resolve(callback(project)) === true) { + deferred.tryResolve(project); + } + } catch (e) { + console.error(e); + } finally { + doneCount++; + } + //if this was the last promise, and we didn't resolve, then resolve with undefined + if (doneCount >= projectCount) { + deferred.tryResolve(undefined); } + }); + return deferred.promise; + } + + public async getSemanticTokens(srcPath: string) { + //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. + const project = await this.findFirstMatchingProject((p) => { + return p.hasFile(srcPath); + }); + + //if we found a project + if (project) { + const result = await Promise.resolve( + project.getSemanticTokens(srcPath) + ); + return result; } } From 76fd82fbd716fed0006da434f908f45f2ba9c1cd Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 2 Jan 2024 15:39:40 -0500 Subject: [PATCH 013/119] Fix diagnostic event flow --- src/lsp/LspProject.ts | 1 + src/lsp/Project.ts | 12 +++---- src/lsp/ProjectManager.ts | 12 +++---- src/lsp/worker/MessageHandler.ts | 2 +- src/lsp/worker/WorkerThreadProject.ts | 11 +++--- src/lsp/worker/WorkerThreadProjectRunner.ts | 39 ++++++++++++++++----- 6 files changed, 48 insertions(+), 29 deletions(-) diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 7b4b09739..7d7e79629 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -44,6 +44,7 @@ export interface LspProject { * @param handler */ on(eventName: 'diagnostics', handler: (data: { diagnostics: LspDiagnostic[] }) => void); + on(eventName: 'all', handler: (eventName: string, data: Record) => void); /** * Release all resources so this file can be safely garbage collected diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 39b372fd8..86dd90393 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -11,10 +11,8 @@ import { Deferred } from '../deferred'; export class Project implements LspProject { /** * Activates this project. Every call to `activate` should completely reset the project, clear all used ram and start from scratch. - * @param options */ public async activate(options: ActivateOptions) { - this.dispose(); this.isActivated = new Deferred(); this.projectPath = options.projectPath; @@ -41,7 +39,6 @@ export class Project implements LspProject { name: 'bsc-language-server', afterProgramValidate: () => { this.emit('diagnostics', { - project: this, diagnostics: this.getDiagnostics() }); } @@ -70,7 +67,7 @@ export class Project implements LspProject { } //flush any diagnostics generated by this initial run - this.emit('diagnostics', { project: this, diagnostics: this.getDiagnostics() }); + this.emit('diagnostics', { diagnostics: this.getDiagnostics() }); this.isActivated.resolve(); } @@ -158,8 +155,7 @@ export class Project implements LspProject { return configFilePath; } else { this.emit('critical-failure', { - message: `Cannot find config file specified in user or workspace settings at '${configFilePath}'`, - project: this + message: `Cannot find config file specified in user or workspace settings at '${configFilePath}'` }); } } @@ -195,8 +191,8 @@ export class Project implements LspProject { }; } - private emit(eventName: 'critical-failure', data: { project: Project; message: string }); - private emit(eventName: 'diagnostics', data: { project: Project; diagnostics: LspDiagnostic[] }); + private emit(eventName: 'critical-failure', data: { message: string }); + private emit(eventName: 'diagnostics', data: { diagnostics: LspDiagnostic[] }); private async emit(eventName: string, data?) { //emit these events on next tick, otherwise they will be processed immediately which could cause issues await util.sleep(0); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index a72083b79..a0c83e7ce 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -67,8 +67,6 @@ export class ProjectManager { /** * Return the first project where the async matcher returns true - * @param callback - * @returns */ private findFirstMatchingProject(callback: (project: LspProject) => boolean | PromiseLike) { const deferred = new Deferred(); @@ -215,12 +213,14 @@ export class ProjectManager { this.projects.push(project); - project.on('diagnostics', (event) => { - this.emit('diagnostics', { ...event, project: project }); + //pipe all project-specific events through our emitter, and include the project reference + project.on('all', (eventName, data) => { + this.emit(eventName as any, { + ...data, + project: project + } as any); }); - //TODO subscribe to various events for this project - await project.activate({ projectPath: config1.projectPath, workspaceFolder: config1.workspaceFolder, diff --git a/src/lsp/worker/MessageHandler.ts b/src/lsp/worker/MessageHandler.ts index 052413a20..5ccbcc58d 100644 --- a/src/lsp/worker/MessageHandler.ts +++ b/src/lsp/worker/MessageHandler.ts @@ -189,6 +189,6 @@ export interface WorkerUpdate { export type WorkerMessage = WorkerRequest | WorkerResponse | WorkerUpdate; -type MethodNames = { +export type MethodNames = { [K in keyof T]: T[K] extends (...args: any[]) => any ? K : never; }[keyof T]; diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 3be386fe5..91682b650 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -109,16 +109,17 @@ export class WorkerThreadProject implements LspProject { */ public configFilePath?: string; - public on(eventName: 'critical-failure', handler: (data: { project: Project; message: string }) => void); + public on(eventName: 'critical-failure', handler: (data: { message: string }) => void); public on(eventName: 'diagnostics', handler: (data: { diagnostics: LspDiagnostic[] }) => MaybePromise); - public on(eventName: string, handler: (payload: any) => void) { - this.emitter.on(eventName, handler); + public on(eventName: 'all', handler: (eventName: string, data: any) => MaybePromise); + public on(eventName: string, handler: (...args: any[]) => MaybePromise) { + this.emitter.on(eventName, handler as any); return () => { - this.emitter.removeListener(eventName, handler); + this.emitter.removeListener(eventName, handler as any); }; } - private emit(eventName: 'critical-failure', data: { project: Project; message: string }); + private emit(eventName: 'critical-failure', data: { message: string }); private emit(eventName: 'diagnostics', data: { diagnostics: LspDiagnostic[] }); private async emit(eventName: string, data?) { //emit these events on next tick, otherwise they will be processed immediately which could cause issues diff --git a/src/lsp/worker/WorkerThreadProjectRunner.ts b/src/lsp/worker/WorkerThreadProjectRunner.ts index d77a73dfc..b2cbad1c3 100644 --- a/src/lsp/worker/WorkerThreadProjectRunner.ts +++ b/src/lsp/worker/WorkerThreadProjectRunner.ts @@ -1,6 +1,6 @@ import type { LspProject } from '../LspProject'; import { Project } from '../Project'; -import type { WorkerMessage } from './MessageHandler'; +import type { MethodNames, WorkerMessage } from './MessageHandler'; import { MessageHandler } from './MessageHandler'; import type { MessagePort } from 'worker_threads'; @@ -8,22 +8,30 @@ import type { MessagePort } from 'worker_threads'; * Runner logic for Running a Project in a worker thread. */ export class WorkerThreadProjectRunner { + //collection of interceptors that will be called when events are fired + private requestInterceptors = {} as Record, (data: any) => any>; + + private project: Project; + + private messageHandler: MessageHandler; + public run(parentPort: MessagePort) { - //make a new instance of the project (which is the same way we run it in the main thread). - const project = new Project(); - const messageHandler = new MessageHandler({ + this.messageHandler = new MessageHandler({ name: 'WorkerThread', port: parentPort, onRequest: async (request: WorkerMessage) => { try { + //if we have a request interceptor registered for this event, call it + this.requestInterceptors[request.name]?.(request.data); + //only the LspProject interface method names will be passed as request names, so just call those functions on the Project class directly - let responseData = await project[request.name](...request.data ?? []); - messageHandler.sendResponse(request, { data: responseData }); + let responseData = await this.project[request.name](...request.data ?? []); + this.messageHandler.sendResponse(request, { data: responseData }); //we encountered a runtime crash. Pass that error along as the response to this request } catch (e) { const error: Error = e as any; - messageHandler.sendResponse(request, { error: error }); + this.messageHandler.sendResponse(request, { error: error }); } }, onUpdate: (update) => { @@ -31,8 +39,21 @@ export class WorkerThreadProjectRunner { } }); - project.on('all', (eventName: string, data: any) => { - messageHandler.sendUpdate(eventName, { + this.requestInterceptors.activate = this.onActivate.bind(this); + } + + /** + * Fired anytime we get an `activate` request from the client. This allows us to clean up the previous project and make a new one + */ + private onActivate() { + //clean up any existing project + this.project.dispose(); + + //make a new instance of the project (which is the same way we run it in the main thread). + this.project = new Project(); + + this.project.on('all', (eventName: string, data: any) => { + this.messageHandler.sendUpdate(eventName, { data: data }); }); From d50d700f95a5b0a7f9bc797fb03f28353358caa7 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 9 Jan 2024 16:29:40 -0500 Subject: [PATCH 014/119] Add basic file change throttling on a per-project basis. --- src/LanguageServer.ts | 49 ++--------- src/lsp/DocumentManager.spec.ts | 90 +++++++++++++++++++++ src/lsp/DocumentManager.ts | 112 ++++++++++++++++++-------- src/lsp/LspProject.ts | 32 +++++++- src/lsp/Project.spec.ts | 6 +- src/lsp/Project.ts | 76 +++++++++++++++++ src/lsp/ProjectManager.spec.ts | 2 +- src/lsp/ProjectManager.ts | 53 ++++++++++++ src/lsp/ReaderWriterManager.ts | 73 +++++++++++++++++ src/lsp/worker/WorkerThreadProject.ts | 75 ++++++++++++++++- 10 files changed, 484 insertions(+), 84 deletions(-) create mode 100644 src/lsp/DocumentManager.spec.ts create mode 100644 src/lsp/ReaderWriterManager.ts diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 3ddfa9dd1..667644009 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -68,12 +68,6 @@ export class LanguageServer implements OnHandler { */ private projectManager: ProjectManager; - - /** - * The number of milliseconds that should be used for language server typing debouncing - */ - private debounceTimeout = 150; - /** * These projects are created on the fly whenever a file is opened that is not included * in any of the workspace-based projects. @@ -140,14 +134,14 @@ export class LanguageServer implements OnHandler { } } - //Register semantic token requestsTODO switch to a more specific connection function call once they actually add it + //Register semantic token requests. TODO switch to a more specific connection function call once they actually add it this.connection.onRequest(SemanticTokensRequest.method, this.onFullSemanticTokens.bind(this)); // The content of a text document has changed. This event is emitted // when the text document is first opened, when its content has changed, // or when document is closed without saving (original contents are sent as a change) // - this.documents.onDidChangeContent(this.validateTextDocument.bind(this)); + this.documents.onDidChangeContent(this.onTextDocumentDidChangeContent.bind(this)); //whenever a document gets closed this.documents.onDidClose(this.onDocumentClose.bind(this)); @@ -586,7 +580,7 @@ export class LanguageServer implements OnHandler { private sendBusyStatus(status: BusyStatus) { this.busyStatusIndex = ++this.busyStatusIndex <= 0 ? 0 : this.busyStatusIndex; - this.connection.sendNotification(NotificationName.busyStatus, { + void this.connection.sendNotification(NotificationName.busyStatus, { status: status, timestamp: Date.now(), index: this.busyStatusIndex, @@ -673,7 +667,7 @@ export class LanguageServer implements OnHandler { * Send a critical failure notification to the client, which should show a notification of some kind */ private sendCriticalFailure(message: string) { - this.connection.sendNotification('critical-failure', message); + void this.connection.sendNotification('critical-failure', message); } /** @@ -950,38 +944,9 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage @TrackBusyStatus - private async validateTextDocument(event: TextDocumentChangeEvent): Promise { - return; - const { document } = event; - //ensure programs are initialized - await this.waitAllProjectFirstRuns(); - - let filePath = URI.parse(document.uri).fsPath; - - try { - - //throttle file processing. first call is run immediately, and then the last call is processed. - await this.keyedThrottler.run(filePath, async () => { - - let documentText = document.getText(); - for (const project of this.getProjects()) { - //only add or replace existing files. All of the files in the project should - //have already been loaded by other means - if (project.builder.program.hasFile(filePath)) { - let rootDir = project.builder.program.options.rootDir ?? project.builder.program.options.cwd; - let dest = rokuDeploy.getDestPath(filePath, project.builder.program.options.files, rootDir); - project.builder.program.setFile({ - src: filePath, - dest: dest - }, documentText); - } - } - // validate all projects - await this.validateAllThrottled(); - }); - } catch (e: any) { - this.sendCriticalFailure(`Critical error parsing/validating ${filePath}: ${e.message}`); - } + private onTextDocumentDidChangeContent(event: TextDocumentChangeEvent) { + const srcPath = URI.parse(event.document.uri).fsPath; + this.projectManager.setFile(srcPath, event.document.getText()); } @TrackBusyStatus diff --git a/src/lsp/DocumentManager.spec.ts b/src/lsp/DocumentManager.spec.ts new file mode 100644 index 000000000..864d65aee --- /dev/null +++ b/src/lsp/DocumentManager.spec.ts @@ -0,0 +1,90 @@ +import { expect } from 'chai'; +import util from '../util'; +import { DocumentManager } from './DocumentManager'; + +describe.only('DocumentManager', () => { + let manager: DocumentManager; + beforeEach(() => { + manager = new DocumentManager({ + delay: 5 + }); + }); + + it('throttles multiple events', async () => { + const actionsPromise = manager.once('flush'); + manager.set('alpha', 'one'); + await util.sleep(1); + manager.set('alpha', 'two'); + await util.sleep(1); + manager.set('alpha', 'three'); + expect( + await actionsPromise + ).to.eql({ + delete: [], + set: [{ + type: 'set', + srcPath: 'alpha', + fileContents: 'three' + }] + }); + }); + + it('any file change delays the first one', async () => { + const actionsPromise = manager.once('flush'); + + manager.set('alpha', 'one'); + await util.sleep(1); + + manager.set('beta', 'two'); + await util.sleep(1); + + manager.set('alpha', 'three'); + await util.sleep(1); + + manager.set('beta', 'four'); + await util.sleep(1); + + expect( + await actionsPromise + ).to.eql({ + delete: [], + set: [{ + type: 'set', + srcPath: 'alpha', + fileContents: 'three' + }, { + type: 'set', + srcPath: 'beta', + fileContents: 'four' + }] + }); + }); + + it('keeps the last-in change', async () => { + manager.set('alpha', 'one'); + manager.delete('alpha'); + expect( + await manager.once('flush') + ).to.eql({ + delete: [{ + type: 'delete', + srcPath: 'alpha' + }], + set: [] + }); + + manager.set('alpha', 'two'); + manager.delete('alpha'); + manager.set('alpha', 'three'); + expect( + await manager.once('flush') + ).to.eql({ + delete: [], + set: [{ + type: 'set', + srcPath: 'alpha', + fileContents: 'three' + }] + }); + }); +}); diff --git a/src/lsp/DocumentManager.ts b/src/lsp/DocumentManager.ts index 744a564fe..280c76f86 100644 --- a/src/lsp/DocumentManager.ts +++ b/src/lsp/DocumentManager.ts @@ -1,30 +1,51 @@ +import * as EventEmitter from 'eventemitter3'; +import type { MaybePromise } from './LspProject'; + /** * Maintains a queued/buffered list of file operations. These operations don't actually do anything on their own. * You need to call the .apply() function and provide an action to operate on them. */ export class DocumentManager { + + constructor( + private options: { delay: number }) { + } + private queue = new Map(); + private timeoutHandle: NodeJS.Timeout; + private throttle() { + if (this.timeoutHandle) { + clearTimeout(this.timeoutHandle); + } + this.timeoutHandle = setTimeout(() => { + this.flush(); + }, this.options.delay); + } + /** * Add/set the contents of a file * @param document */ - public set(document: Document) { - if (this.queue.has(document.paths.src)) { - this.queue.delete(document.paths.src); + public set(srcPath: string, fileContents: string) { + if (this.queue.has(srcPath)) { + this.queue.delete(srcPath); } - this.queue.set(document.paths.src, { action: 'set', document: document }); + this.queue.set(srcPath, { + type: 'set', + srcPath: srcPath, + fileContents: fileContents + }); + this.throttle(); } /** * Delete a file * @param document */ - public delete(document: Document) { - if (this.queue.has(document.paths.src)) { - this.queue.delete(document.paths.src); - } - this.queue.set(document.paths.src, { action: 'delete', document: document }); + public delete(srcPath: string) { + this.queue.delete(srcPath); + this.queue.set(srcPath, { type: 'delete', srcPath: srcPath }); } /** @@ -34,35 +55,58 @@ export class DocumentManager { return this.queue.size > 0; } - /** - * Indicates whether we are currently in the middle of an `apply()` session or not - */ - public isBlocked = false; + private flush() { + const event: FlushEvent = { + actions: [...this.queue.values()] + }; + this.queue.clear(); - /** - * Get all of the pending documents and clear the queue - */ - public async apply(action: (actions: DocumentAction[]) => any): Promise { - this.isBlocked = true; - try { - const documentActions = [...this.queue.values()]; - const result = await Promise.resolve(action(documentActions)); - return result; - } finally { - this.isBlocked = false; - } + this.emitSync('flush', event); + } + + public once(eventName: 'flush'): Promise; + public once(eventName: string): Promise { + return new Promise((resolve) => { + const off = this.on(eventName as any, (data) => { + off(); + resolve(data); + }); + }); + } + + public on(eventName: 'flush', handler: (data: any) => MaybePromise); + public on(eventName: string, handler: (...args: any[]) => MaybePromise) { + this.emitter.on(eventName, handler as any); + return () => { + this.emitter.removeListener(eventName, handler as any); + }; + } + + private emitSync(eventName: 'flush', data: FlushEvent); + private emitSync(eventName: string, data?) { + this.emitter.emit(eventName, data); + } + + private emitter = new EventEmitter(); + + public dispose() { + this.queue = new Map(); + this.emitter.removeAllListeners(); } } -export interface DocumentAction { - action: 'set' | 'delete'; - document: Document; +export interface SetDocumentAction { + type: 'set'; + srcPath: string; + fileContents: string; +} +export interface DeleteDocumentAction { + type: 'delete'; + srcPath: string; } -export interface Document { - paths: { - src: string; - dest: string; - }; - getText: () => string; +export type DocumentAction = SetDocumentAction | DeleteDocumentAction; + +export interface FlushEvent { + actions: DocumentAction[]; } diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 7d7e79629..232ba8608 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,5 +1,6 @@ import type { Diagnostic } from 'vscode-languageserver'; import type { SemanticToken } from '../interfaces'; +import type { BsConfig } from '../BsConfig'; /** * Defines the contract between the ProjectManager and the main or worker thread Project classes @@ -22,6 +23,22 @@ export interface LspProject { */ activate(options: ActivateOptions): MaybePromise; + /** + * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, + * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project + */ + validate(): MaybePromise; + + /** + * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. + */ + getOptions(): MaybePromise; + + /** + * Get the list of all file paths that are currently loaded in the project + */ + getFilePaths(): MaybePromise; + /** * Get the list of all diagnostics from this project */ @@ -34,10 +51,23 @@ export interface LspProject { getSemanticTokens(srcPath: string): MaybePromise; /** - * Does this project have the specified filie + * Does this project have the specified file. Should only be called after `.activate()` has completed. */ hasFile(srcPath: string): MaybePromise; + /** + * Add or replace the in-memory contents of the file at the specified path. This is typically called as the user is typing. + * @param srcPath absolute path to the file + * @param fileContents the contents of the file + */ + setFile(srcPath: string, fileContents: string): MaybePromise; + + /** + * Remove the in-memory file at the specified path. This is typically called when the user (or file system watcher) triggers a file delete + * @param srcPath absolute path to the file + */ + removeFile(srcPath: string): MaybePromise; + /** * An event that is emitted anytime the diagnostics for the project have changed (typically after a validate cycle has finished) * @param eventName diff --git a/src/lsp/Project.spec.ts b/src/lsp/Project.spec.ts index 8e0dde431..b0bcf6897 100644 --- a/src/lsp/Project.spec.ts +++ b/src/lsp/Project.spec.ts @@ -27,15 +27,15 @@ describe('Project', () => { it('emits events', async () => { const stub = sinon.stub(); const off = project.on('diagnostics', stub); - await project['emit']('diagnostics', { project: project, diagnostics: [] }); + await project['emit']('diagnostics', { diagnostics: [] }); expect(stub.callCount).to.eql(1); - await project['emit']('diagnostics', { project: project, diagnostics: [] }); + await project['emit']('diagnostics', { diagnostics: [] }); expect(stub.callCount).to.eql(2); off(); - await project['emit']('diagnostics', { project: project, diagnostics: [] }); + await project['emit']('diagnostics', { diagnostics: [] }); expect(stub.callCount).to.eql(2); }); }); diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 86dd90393..b04496a0a 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -7,6 +7,9 @@ import type { CompilerPlugin } from '../interfaces'; import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; +import { rokuDeploy } from 'roku-deploy'; +import { DocumentManager } from './DocumentManager'; +import { ReaderWriterManager } from './ReaderWriterManager'; export class Project implements LspProject { /** @@ -72,6 +75,21 @@ export class Project implements LspProject { this.isActivated.resolve(); } + /** + * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, + * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project + */ + public validate() { + this.builder.program.validate(); + } + + /** + * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. + */ + public getOptions() { + return this.builder.program.options; + } + /** * Gets resolved when the project has finished activating */ @@ -105,6 +123,37 @@ export class Project implements LspProject { return this.builder.program.hasFile(srcPath); } + /** + * Set new contents for a file. This is safe to call any time. Changes will be queued and flushed at the correct times + * during the program's lifecycle flow + * @param srcPath absolute source path of the file + * @param fileContents the text contents of the file + */ + public setFile(srcPath: string, fileContents: string) { + this.builder.program.setFile( + { + src: srcPath, + dest: rokuDeploy.getDestPath(srcPath, this.getFilePaths(), this.builder.program.options.rootDir) + }, + fileContents + ); + } + + /** + * Remove the in-memory file at the specified path. This is typically called when the user (or file system watcher) triggers a file delete + * @param srcPath absolute path to the file + */ + public removeFile(srcPath: string) { + this.builder.program.removeFile(srcPath); + } + + /** + * Get the list of all file paths that are currently loaded in the project + */ + public getFilePaths() { + return Object.keys(this.builder.program.files).sort(); + } + /** * Get the full list of semantic tokens for the given file path * @param srcPath absolute path to the source file @@ -212,3 +261,30 @@ export class Project implements LspProject { } } } + +/** + * An annotation used to wrap the method in a readerWriter.write() call + */ +function WriteLock(target: any, propertyKey: string, descriptor: PropertyDescriptor) { + let originalMethod = descriptor.value; + + //wrapping the original method + descriptor.value = function value(this: Project, ...args: any[]) { + return (this as any).readerWriter.write(() => { + return originalMethod.apply(this, args); + }, originalMethod.name); + }; +} +/** + * An annotation used to wrap the method in a readerWriter.read() call + */ +function ReadLock(target: any, propertyKey: string, descriptor: PropertyDescriptor) { + let originalMethod = descriptor.value; + + //wrapping the original method + descriptor.value = function value(this: Project, ...args: any[]) { + return (this as any).readerWriter.read(() => { + return originalMethod.apply(this, args); + }, originalMethod.name); + }; +} diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index 9922ef43c..cdf576286 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -249,7 +249,7 @@ describe('ProjectManager', () => { }); }); - describe.only('raceUntil', () => { + describe('raceUntil', () => { beforeEach(() => { }); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index a0c83e7ce..14b2a3d1b 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -7,16 +7,59 @@ import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; import type { Position } from 'vscode-languageserver'; import { Deferred } from '../deferred'; +import type { FlushEvent } from './DocumentManager'; +import { DocumentManager } from './DocumentManager'; /** * Manages all brighterscript projects for the language server */ export class ProjectManager { + /** * Collection of all projects */ public projects: LspProject[] = []; + private documentManager = new DocumentManager({ + delay: 150 + }); + + + constructor() { + this.documentManager.on('flush', (event) => { + void this.applyDocumentChanges(event); + }); + } + + /** + * Apply all of the queued document changes. This should only be called as a result of the documentManager flushing changes, and never called manually + * @param event the document changes that have occurred since the last time we applied + */ + private async applyDocumentChanges(event: FlushEvent) { + //apply all of the document actions to each project in parallel + await Promise.all(this.projects.map(async (project) => { + await Promise.all(event.actions.map(async (action) => { + if (action.type === 'set') { + await project.setFile(action.srcPath, action.fileContents); + } else if (action.type === 'delete') { + await project.removeFile(action.srcPath); + } + })); + + //now that all the files have been sent, validate the project (which will trigger downstream diagnostics to flow) + await this.validateProject(project); + })); + } + + /** + * Validate the given project. This wraps the call in locks to ensure other actions will wait + * @param project + */ + private async validateProject(project) { + await project.validate(); + } + + /** * Given a list of all desired projects, create any missing projects and destroy and projects that are no longer available * Treat workspaces that don't have a bsconfig.json as a project. @@ -65,6 +108,16 @@ export class ProjectManager { ); } + /** + * Set new contents for a file. This is safe to call any time. Changes will be queued and flushed at the correct times + * during the program's lifecycle flow + * @param srcPath absolute source path of the file + * @param fileContents the text contents of the file + */ + public setFile(srcPath: string, fileContents: string) { + this.documentManager.set(srcPath, fileContents); + } + /** * Return the first project where the async matcher returns true */ diff --git a/src/lsp/ReaderWriterManager.ts b/src/lsp/ReaderWriterManager.ts new file mode 100644 index 000000000..705906173 --- /dev/null +++ b/src/lsp/ReaderWriterManager.ts @@ -0,0 +1,73 @@ +import { Deferred } from '../deferred'; +import type { MaybePromise } from './LspProject'; + +/** + * Manages multiple readers and writers, and ensures that no readers are reading while a writer is writing. + * This is useful when multiple file changes show up but we also got a completions request, so we need to wait + * until the files have been written and the program is validated before executing the completions request + */ +export class ReaderWriterManager { + + private readers: Array; + + private writers: Array; + + /** + * Register a read action + * @param action + */ + public read(action: Action) { + const reader = { + action: action, + deferred: new Deferred() + }; + this.readers.push(reader); + void this.execute(); + return reader.deferred.promise; + } + + /** + * Register a write action + * @param action + */ + public write(action: Action) { + const writer = { + action: action, + deferred: new Deferred() + }; + this.writers.push(writer); + void this.execute(); + return writer.deferred.promise; + } + + private async execute() { + let item: ReaderWriter; + if (this.writers.length > 0) { + item = this.writers.pop(); + } else if (this.readers.length > 0) { + item = this.readers.pop(); + + //there are no more readers or writers, so quit. + } else { + return; + } + //execute the item + try { + const result = await Promise.resolve( + item.action() + ); + item.deferred.resolve(result); + } catch (e) { + item.deferred.reject(e); + } + //execute the next action (if there is one) + void this.execute(); + } +} + +type Action = () => MaybePromise; + +interface ReaderWriter { + action: Action; + deferred: Deferred; +} diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 91682b650..341421f4d 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -7,9 +7,9 @@ import type { LspDiagnostic, MaybePromise } from '../LspProject'; import { type ActivateOptions, type LspProject } from '../LspProject'; import { isMainThread, parentPort } from 'worker_threads'; import { WorkerThreadProjectRunner } from './WorkerThreadProjectRunner'; -import type { Project } from '../Project'; import { WorkerPool } from './WorkerPool'; import type { SemanticToken } from '../../interfaces'; +import type { BsConfig } from '../../BsConfig'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -47,19 +47,88 @@ export class WorkerThreadProject implements LspProject { onRequest: this.processRequest.bind(this), onUpdate: this.processUpdate.bind(this) }); + await this.messageHandler.sendRequest('activate', { data: [options] }); + + //populate a few properties with data from the thread so we can use them for some synchronous checks + this.filePaths = await this.getFilePaths(); + this.options = await this.getOptions(); } + /** + * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, + * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project + */ + public async validate() { + const response = await this.messageHandler.sendRequest('validate'); + return response.data; + } + + /** + * A local copy of all the file paths loaded in this program. This needs to stay in sync with any files we add/delete in the worker thread, + * so we can keep doing in-process `.hasFile()` checks + */ + private filePaths: string[]; + public async getDiagnostics() { const response = await this.messageHandler.sendRequest('getDiagnostics'); return response.data; } - public async hasFile(srcPath: string) { - const response = await this.messageHandler.sendRequest('hasFile'); + /** + * Does this project have the specified file. Should only be called after `.activate()` has finished/ + */ + public hasFile(srcPath: string) { + return this.filePaths.includes(srcPath); + } + + /** + * Set new contents for a file. This is safe to call any time. Changes will be queued and flushed at the correct times + * during the program's lifecycle flow + * @param srcPath absolute source path of the file + * @param fileContents the text contents of the file + */ + public async setFile(srcPath: string, fileContents: string) { + const response = await this.messageHandler.sendRequest('setFile', { + data: [srcPath, fileContents] + }); + return response.data; + } + + /** + * Remove the in-memory file at the specified path. This is typically called when the user (or file system watcher) triggers a file delete + * @param srcPath absolute path to the file + */ + public async removeFile(srcPath: string) { + const response = await this.messageHandler.sendRequest('removeFile', { + data: [srcPath] + }); return response.data; } + /** + * Get the list of all file paths that are currently loaded in the project + */ + public async getFilePaths() { + return (await this.messageHandler.sendRequest('getFilePaths')).data; + } + + /** + * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. + */ + public async getOptions() { + return (await this.messageHandler.sendRequest('getOptions')).data; + } + + /** + * A local reference to the bsconfig this project was built with. Should only be accessed after `.activate()` has completed. + */ + private options: BsConfig; + + public get rootDir() { + return this.options.rootDir; + } + /** * Get the full list of semantic tokens for the given file path * @param srcPath absolute path to the source file From 101b98c79c34a4485d000da3d56bcb019520930c Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Thu, 11 Jan 2024 14:21:32 -0500 Subject: [PATCH 015/119] Run program.validate() in async chunks to not starve CPU. Support cancelling validate --- src/LanguageServer.ts | 9 ++-- src/Logger.ts | 18 ++++--- src/Program.spec.ts | 2 +- src/Program.ts | 86 +++++++++++++++++++-------------- src/common/Sequencer.spec.ts | 21 ++++++++ src/common/Sequencer.ts | 85 ++++++++++++++++++++++++++++++++ src/lsp/DocumentManager.spec.ts | 58 +++++++++++----------- src/lsp/LspProject.ts | 4 +- src/lsp/Project.ts | 10 ++-- src/lsp/ProjectManager.ts | 27 ++++++++++- src/util.ts | 10 +++- 11 files changed, 246 insertions(+), 84 deletions(-) create mode 100644 src/common/Sequencer.spec.ts create mode 100644 src/common/Sequencer.ts diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 667644009..95318adc4 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -178,10 +178,10 @@ export class LanguageServer implements OnHandler { // }, // documentSymbolProvider: true, // workspaceSymbolProvider: true, - semanticTokensProvider: { - legend: semanticTokensLegend, - full: true - } as SemanticTokensOptions + // semanticTokensProvider: { + // legend: semanticTokensLegend, + // full: true + // } as SemanticTokensOptions // referencesProvider: true, // codeActionProvider: { // codeActionKinds: [CodeActionKind.Refactor] @@ -946,6 +946,7 @@ export class LanguageServer implements OnHandler { @TrackBusyStatus private onTextDocumentDidChangeContent(event: TextDocumentChangeEvent) { const srcPath = URI.parse(event.document.uri).fsPath; + console.log('setFile', srcPath); this.projectManager.setFile(srcPath, event.document.getText()); } diff --git a/src/Logger.ts b/src/Logger.ts index cac73702d..bd3a88169 100644 --- a/src/Logger.ts +++ b/src/Logger.ts @@ -125,10 +125,10 @@ export class Logger { * Writes to the log (if logLevel matches), and also times how long the action took to occur. * `action` is called regardless of logLevel, so this function can be used to nicely wrap * pieces of functionality. - * The action function also includes two parameters, `pause` and `resume`, which can be used to improve timings by focusing only on - * the actual logic of that action. + * The action function also includes two parameters called `pause` and `resume`, which can be used to improve timings by focusing only on + * the actual logic of that action. The third parameter is called `cancel`, and will prevent the log function from being run */ - time(logLevel: LogLevel, messages: any[], action: (pause: () => void, resume: () => void) => T): T { + time(logLevel: LogLevel, messages: any[], action: (pause: () => void, resume: () => void, cancel: () => void) => T): T { //call the log if loglevel is in range if (this._logLevel >= logLevel) { const stopwatch = new Stopwatch(); @@ -138,15 +138,21 @@ export class Logger { this[logLevelString](...messages ?? []); this.indent += ' '; + let isCanceled = false; + stopwatch.start(); //execute the action - const result = action(stopwatch.stop.bind(stopwatch), stopwatch.start.bind(stopwatch)) as any; + const result = action(stopwatch.stop.bind(stopwatch), stopwatch.start.bind(stopwatch), () => { + isCanceled = true; + }) as any; //return a function to call when the timer is complete const done = () => { stopwatch.stop(); this.indent = this.indent.substring(2); - this[logLevelString](...messages ?? [], `finished. (${chalk.blue(stopwatch.getDurationText())})`); + if (!isCanceled) { + this[logLevelString](...messages ?? [], `finished. (${chalk.blue(stopwatch.getDurationText())})`); + } }; //if this is a promise, wait for it to resolve and then return the original result @@ -160,7 +166,7 @@ export class Logger { return result; } } else { - return action(noop, noop); + return action(noop, noop, noop); } } } diff --git a/src/Program.spec.ts b/src/Program.spec.ts index 04b405980..feac43a99 100644 --- a/src/Program.spec.ts +++ b/src/Program.spec.ts @@ -24,7 +24,7 @@ import { tempDir, rootDir, stagingDir } from './testHelpers.spec'; let sinon = sinonImport.createSandbox(); -describe('Program', () => { +describe.only('Program', () => { let program: Program; beforeEach(() => { diff --git a/src/Program.ts b/src/Program.ts index 27022543f..7c0f775ac 100644 --- a/src/Program.ts +++ b/src/Program.ts @@ -1,8 +1,8 @@ import * as assert from 'assert'; import * as fsExtra from 'fs-extra'; import * as path from 'path'; -import type { CodeAction, CompletionItem, Position, Range, SignatureInformation, Location } from 'vscode-languageserver'; -import { CompletionItemKind } from 'vscode-languageserver'; +import type { CodeAction, CompletionItem, Position, Range, SignatureInformation, Location, CancellationToken } from 'vscode-languageserver'; +import { CancellationTokenSource, CompletionItemKind } from 'vscode-languageserver'; import type { BsConfig } from './BsConfig'; import { Scope } from './Scope'; import { DiagnosticMessages } from './DiagnosticMessages'; @@ -29,6 +29,7 @@ import type { Statement } from './parser/AstNode'; import { CallExpressionInfo } from './bscPlugin/CallExpressionInfo'; import { SignatureHelpUtil } from './bscPlugin/SignatureHelpUtil'; import { DiagnosticSeverityAdjuster } from './DiagnosticSeverityAdjuster'; +import { Sequencer } from './common/Sequencer'; const startOfSourcePkgPath = `source${path.sep}`; const bslibNonAliasedRokuModulesPkgPath = s`source/roku_modules/rokucommunity_bslib/bslib.brs`; @@ -658,45 +659,56 @@ export class Program { /** * Traverse the entire project, and validate all scopes */ - public validate() { - this.logger.time(LogLevel.log, ['Validating project'], () => { - this.diagnostics = []; - this.plugins.emit('beforeProgramValidate', this); - - //validate every file - for (const file of Object.values(this.files)) { - //for every unvalidated file, validate it - if (!file.isValidated) { - this.plugins.emit('beforeFileValidate', { - program: this, - file: file - }); - - //emit an event to allow plugins to contribute to the file validation process - this.plugins.emit('onFileValidate', { - program: this, - file: file - }); - //call file.validate() IF the file has that function defined - file.validate?.(); - file.isValidated = true; - - this.plugins.emit('afterFileValidate', file); - } - } + public validate(): void; + public validate(options: { async: false; cancellationToken?: CancellationToken }): void; + public validate(options: { async: true; cancellationToken?: CancellationToken }): Promise; + public validate(options?: { async?: boolean; cancellationToken?: CancellationToken }) { + return this.logger.time(LogLevel.log, ['Validating project'], (start, stop, cancel) => { + + const sequencer = new Sequencer({ + name: 'program.validate', + async: options?.async ?? false, + cancellationToken: options?.cancellationToken ?? new CancellationTokenSource().token + }); - this.logger.time(LogLevel.info, ['Validate all scopes'], () => { - for (let scopeName in this.scopes) { - let scope = this.scopes[scopeName]; + //for every unvalidated file, validate it + return sequencer + .once(() => { + this.diagnostics = []; + this.plugins.emit('beforeProgramValidate', this); + }) + .forEach(Object.values(this.files), (file) => { + if (!file.isValidated) { + this.plugins.emit('beforeFileValidate', { + program: this, + file: file + }); + + //emit an event to allow plugins to contribute to the file validation process + this.plugins.emit('onFileValidate', { + program: this, + file: file + }); + //call file.validate() IF the file has that function defined + file.validate?.(); + file.isValidated = true; + + this.plugins.emit('afterFileValidate', file); + } + }) + .forEach(Object.values(this.scopes), (scope) => { scope.linkSymbolTable(); scope.validate(); scope.unlinkSymbolTable(); - } - }); - - this.detectDuplicateComponentNames(); - - this.plugins.emit('afterProgramValidate', this); + }) + .once(() => { + this.detectDuplicateComponentNames(); + this.plugins.emit('afterProgramValidate', this); + }) + .onCancel(() => { + cancel(); + }) + .run(); }); } diff --git a/src/common/Sequencer.spec.ts b/src/common/Sequencer.spec.ts new file mode 100644 index 000000000..fac02634c --- /dev/null +++ b/src/common/Sequencer.spec.ts @@ -0,0 +1,21 @@ +import { CancellationTokenSource } from 'vscode-languageserver-protocol'; +import { Sequencer } from './Sequencer'; +import { expect } from '../chai-config.spec'; + +describe('Sequencer', () => { + it('cancels when asked', () => { + const cancellationTokenSource = new CancellationTokenSource(); + const values = []; + void new Sequencer({ + name: 'test', + cancellationToken: cancellationTokenSource.token + }).forEach([1, 2, 3], (i) => { + values.push(i); + if (i === 2) { + cancellationTokenSource.cancel(); + } + }).run(); + + expect(values).to.eql([1, 2]); + }); +}); diff --git a/src/common/Sequencer.ts b/src/common/Sequencer.ts new file mode 100644 index 000000000..456b310e7 --- /dev/null +++ b/src/common/Sequencer.ts @@ -0,0 +1,85 @@ +import type { CancellationToken } from 'vscode-languageserver-protocol'; +import { util } from '../util'; +import { EventEmitter } from 'eventemitter3'; + +/** + * Supports running a series of actions in sequence, either synchronously or asynchronously + */ +export class Sequencer { + constructor( + private options: { + name: string; + async?: boolean; + cancellationToken?: CancellationToken; + } + ) { + + } + + // eslint-disable-next-line @typescript-eslint/ban-types + private actions: Array<{ args: any[]; func: Function }> = []; + + public forEach(items: T[], func: (item: T) => any) { + for (const item of items) { + this.actions.push({ + args: [item], + func: func + }); + } + return this; + } + + private emitter = new EventEmitter(); + + public onCancel(callback: () => void) { + this.emitter.on('cancel', callback); + return this; + } + + public once(func: () => any) { + this.actions.push({ + args: [], + func: func + }); + return this; + } + + /** + * Actually run the sequence + */ + public run() { + if (this.options?.async) { + return this.runAsync(); + } else { + return this.runSync(); + } + } + + private async runAsync() { + for (const action of this.actions) { + //register a very short timeout between every action so we don't hog the CPU + await util.sleep(1); + + //if the cancellation token has asked us to cancel, then stop processing now + if (this.options.cancellationToken?.isCancellationRequested) { + return this.handleCancel(); + } + action.func(...action.args); + } + } + + private runSync() { + for (const action of this.actions) { + //if the cancellation token has asked us to cancel, then stop processing now + if (this.options.cancellationToken.isCancellationRequested) { + return this.handleCancel(); + } + action.func(...action.args); + } + } + + private handleCancel() { + console.log(`Cancelling sequence ${this.options.name}`); + this.emitter.emit('cancel'); + } +} diff --git a/src/lsp/DocumentManager.spec.ts b/src/lsp/DocumentManager.spec.ts index 864d65aee..c3babaf6c 100644 --- a/src/lsp/DocumentManager.spec.ts +++ b/src/lsp/DocumentManager.spec.ts @@ -20,12 +20,13 @@ describe.only('DocumentManager', () => { expect( await actionsPromise ).to.eql({ - delete: [], - set: [{ - type: 'set', - srcPath: 'alpha', - fileContents: 'three' - }] + actions: [ + { + type: 'set', + srcPath: 'alpha', + fileContents: 'three' + } + ] }); }); @@ -47,16 +48,17 @@ describe.only('DocumentManager', () => { expect( await actionsPromise ).to.eql({ - delete: [], - set: [{ - type: 'set', - srcPath: 'alpha', - fileContents: 'three' - }, { - type: 'set', - srcPath: 'beta', - fileContents: 'four' - }] + actions: [ + { + type: 'set', + srcPath: 'alpha', + fileContents: 'three' + }, { + type: 'set', + srcPath: 'beta', + fileContents: 'four' + } + ] }); }); @@ -66,11 +68,12 @@ describe.only('DocumentManager', () => { expect( await manager.once('flush') ).to.eql({ - delete: [{ - type: 'delete', - srcPath: 'alpha' - }], - set: [] + actions: [ + { + type: 'delete', + srcPath: 'alpha' + } + ] }); manager.set('alpha', 'two'); @@ -79,12 +82,13 @@ describe.only('DocumentManager', () => { expect( await manager.once('flush') ).to.eql({ - delete: [], - set: [{ - type: 'set', - srcPath: 'alpha', - fileContents: 'three' - }] + actions: [ + { + type: 'set', + srcPath: 'alpha', + fileContents: 'three' + } + ] }); }); }); diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 232ba8608..6e6a1be39 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,4 +1,4 @@ -import type { Diagnostic } from 'vscode-languageserver'; +import type { CancellationToken, Diagnostic } from 'vscode-languageserver'; import type { SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; @@ -27,7 +27,7 @@ export interface LspProject { * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project */ - validate(): MaybePromise; + validate(options: { cancellationToken: CancellationToken }): Promise; /** * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index b04496a0a..a7eda5d20 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -8,8 +8,7 @@ import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; -import { DocumentManager } from './DocumentManager'; -import { ReaderWriterManager } from './ReaderWriterManager'; +import type { CancellationToken } from 'vscode-languageserver-protocol'; export class Project implements LspProject { /** @@ -79,8 +78,11 @@ export class Project implements LspProject { * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project */ - public validate() { - this.builder.program.validate(); + public async validate(options: { cancellationToken: CancellationToken }) { + await this.builder.program.validate({ + async: true, + ...options + }); } /** diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 14b2a3d1b..060c36ac0 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -6,9 +6,11 @@ import type { LspDiagnostic, LspProject, MaybePromise } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; import type { Position } from 'vscode-languageserver'; +import { CancellationTokenSource } from 'vscode-languageserver'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; +import { Cache } from '../Cache'; /** * Manages all brighterscript projects for the language server @@ -38,6 +40,12 @@ export class ProjectManager { private async applyDocumentChanges(event: FlushEvent) { //apply all of the document actions to each project in parallel await Promise.all(this.projects.map(async (project) => { + + //cancel any active validation that's going on + console.log('cancelValidate'); + this.cancelValidateProject(project); + + //apply every file event to this project await Promise.all(event.actions.map(async (action) => { if (action.type === 'set') { await project.setFile(action.srcPath, action.fileContents); @@ -51,12 +59,27 @@ export class ProjectManager { })); } + private pendingValidations = new Cache(); + /** * Validate the given project. This wraps the call in locks to ensure other actions will wait * @param project */ - private async validateProject(project) { - await project.validate(); + private async validateProject(project: LspProject) { + const cancellationToken = new CancellationTokenSource(); + this.pendingValidations.getOrAdd(project, () => []).push(cancellationToken); + await project.validate({ + cancellationToken: cancellationToken.token + }); + } + + private cancelValidateProject(project: LspProject) { + const tokens = this.pendingValidations.getOrAdd(project, () => []); + + //remove all the tokens from the list, and cancel each one of them + for (const token of tokens.splice(0, tokens.length)) { + token.cancel(); + } } diff --git a/src/util.ts b/src/util.ts index 5637fcfbf..8690ed599 100644 --- a/src/util.ts +++ b/src/util.ts @@ -4,7 +4,7 @@ import type { ParseError } from 'jsonc-parser'; import { parse as parseJsonc, printParseErrorCode } from 'jsonc-parser'; import * as path from 'path'; import { rokuDeploy, DefaultFiles, standardizePath as rokuDeployStandardizePath } from 'roku-deploy'; -import type { Diagnostic, Position, Range, Location } from 'vscode-languageserver'; +import type { Diagnostic, Position, Range, Location, CancellationToken } from 'vscode-languageserver'; import { URI } from 'vscode-uri'; import * as xml2js from 'xml2js'; import type { BsConfig } from './BsConfig'; @@ -1511,6 +1511,14 @@ export class Util { } } } + + /** + * Run a series of actions in chunks. This allows us to run them sequentially or each operation on nexttick to prevent starving the CPU + * @param options + */ + public runInChunks(options: { async: boolean; cancel: CancellationToken; actions: Array<{ collection: any[]; action: (item: any) => any }> }) { + + } } /** From 11dd7fe1c0e22186bfa44acae058a2552493835c Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Thu, 11 Jan 2024 16:44:47 -0500 Subject: [PATCH 016/119] Better handling of worker thread errors --- src/LanguageServer.ts | 3 --- src/lsp/worker/MessageHandler.ts | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 95318adc4..288a7a2b2 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -17,7 +17,6 @@ import type { SignatureHelp, SignatureHelpParams, CodeActionParams, - SemanticTokensOptions, SemanticTokens, SemanticTokensParams, TextDocumentChangeEvent, @@ -36,8 +35,6 @@ import { FileChangeType, ProposedFeatures, TextDocuments, - TextDocumentSyncKind, - CodeActionKind } from 'vscode-languageserver/node'; import { URI } from 'vscode-uri'; import { TextDocument } from 'vscode-languageserver-textdocument'; diff --git a/src/lsp/worker/MessageHandler.ts b/src/lsp/worker/MessageHandler.ts index 5ccbcc58d..118fdeed1 100644 --- a/src/lsp/worker/MessageHandler.ts +++ b/src/lsp/worker/MessageHandler.ts @@ -87,10 +87,8 @@ export class MessageHandler> { this.port.postMessage(request); const response = await responsePromise; if ('error' in response) { - const error = this.objectToError(response.error); - (error as any)._response = response; //throw the error so it causes a rejected promise (like we'd expect) - throw error; + throw new Error(`Worker thread encountered an error: ${JSON.stringify(response.error.stack)}`); } return response; } From be16e9e1bc50977577a8035758157337291cff97 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Thu, 11 Jan 2024 17:00:19 -0500 Subject: [PATCH 017/119] Better time tracking for cancelling validations. --- src/Logger.ts | 22 ++++++++++ src/Program.ts | 92 +++++++++++++++++++++-------------------- src/common/Sequencer.ts | 43 +++++++++++++------ 3 files changed, 100 insertions(+), 57 deletions(-) diff --git a/src/Logger.ts b/src/Logger.ts index bd3a88169..4e714a117 100644 --- a/src/Logger.ts +++ b/src/Logger.ts @@ -121,6 +121,28 @@ export class Logger { } } + /** + * Writes to the log (if logLevel matches), and also provides a function that can be called to mark the end of a time. + * You can override the action if, for example, the operation was cancelled instead of finished. + */ + timeStart(logLevel: LogLevel, ...messages: any[]) { + //call the log if loglevel is in range + if (this._logLevel >= logLevel) { + const stopwatch = new Stopwatch(); + const logLevelString = LogLevel[logLevel]; + + //write the initial log + this[logLevelString](...messages ?? []); + + stopwatch.start(); + + return (status = 'finished') => { + stopwatch.stop(); + this[logLevelString](...messages ?? [], `${status}. (${chalk.blue(stopwatch.getDurationText())})`); + }; + } + } + /** * Writes to the log (if logLevel matches), and also times how long the action took to occur. * `action` is called regardless of logLevel, so this function can be used to nicely wrap diff --git a/src/Program.ts b/src/Program.ts index 7c0f775ac..b43454295 100644 --- a/src/Program.ts +++ b/src/Program.ts @@ -663,53 +663,55 @@ export class Program { public validate(options: { async: false; cancellationToken?: CancellationToken }): void; public validate(options: { async: true; cancellationToken?: CancellationToken }): Promise; public validate(options?: { async?: boolean; cancellationToken?: CancellationToken }) { - return this.logger.time(LogLevel.log, ['Validating project'], (start, stop, cancel) => { + const timeEnd = this.logger.timeStart(LogLevel.log, 'Validating project'); - const sequencer = new Sequencer({ - name: 'program.validate', - async: options?.async ?? false, - cancellationToken: options?.cancellationToken ?? new CancellationTokenSource().token - }); - - //for every unvalidated file, validate it - return sequencer - .once(() => { - this.diagnostics = []; - this.plugins.emit('beforeProgramValidate', this); - }) - .forEach(Object.values(this.files), (file) => { - if (!file.isValidated) { - this.plugins.emit('beforeFileValidate', { - program: this, - file: file - }); - - //emit an event to allow plugins to contribute to the file validation process - this.plugins.emit('onFileValidate', { - program: this, - file: file - }); - //call file.validate() IF the file has that function defined - file.validate?.(); - file.isValidated = true; - - this.plugins.emit('afterFileValidate', file); - } - }) - .forEach(Object.values(this.scopes), (scope) => { - scope.linkSymbolTable(); - scope.validate(); - scope.unlinkSymbolTable(); - }) - .once(() => { - this.detectDuplicateComponentNames(); - this.plugins.emit('afterProgramValidate', this); - }) - .onCancel(() => { - cancel(); - }) - .run(); + const sequencer = new Sequencer({ + name: 'program.validate', + async: options?.async ?? false, + cancellationToken: options?.cancellationToken ?? new CancellationTokenSource().token }); + + //for every unvalidated file, validate it + return sequencer + .once(() => { + this.diagnostics = []; + this.plugins.emit('beforeProgramValidate', this); + }) + .forEach(Object.values(this.files), (file) => { + if (!file.isValidated) { + this.plugins.emit('beforeFileValidate', { + program: this, + file: file + }); + + //emit an event to allow plugins to contribute to the file validation process + this.plugins.emit('onFileValidate', { + program: this, + file: file + }); + //call file.validate() IF the file has that function defined + file.validate?.(); + file.isValidated = true; + + this.plugins.emit('afterFileValidate', file); + } + }) + .forEach(Object.values(this.scopes), (scope) => { + scope.linkSymbolTable(); + scope.validate(); + scope.unlinkSymbolTable(); + }) + .once(() => { + this.detectDuplicateComponentNames(); + this.plugins.emit('afterProgramValidate', this); + }) + .onCancel(() => { + timeEnd('cancelled'); + }) + .onComplete(() => { + timeEnd(); + }) + .run(); } /** diff --git a/src/common/Sequencer.ts b/src/common/Sequencer.ts index 456b310e7..1e954fcf7 100644 --- a/src/common/Sequencer.ts +++ b/src/common/Sequencer.ts @@ -36,6 +36,11 @@ export class Sequencer { return this; } + public onComplete(callback: () => void) { + this.emitter.on('complete', callback); + return this; + } + public once(func: () => any) { this.actions.push({ args: [], @@ -56,25 +61,35 @@ export class Sequencer { } private async runAsync() { - for (const action of this.actions) { - //register a very short timeout between every action so we don't hog the CPU - await util.sleep(1); + try { + for (const action of this.actions) { + //register a very short timeout between every action so we don't hog the CPU + await util.sleep(1); - //if the cancellation token has asked us to cancel, then stop processing now - if (this.options.cancellationToken?.isCancellationRequested) { - return this.handleCancel(); + //if the cancellation token has asked us to cancel, then stop processing now + if (this.options.cancellationToken?.isCancellationRequested) { + return this.handleCancel(); + } + action.func(...action.args); } - action.func(...action.args); + } finally { + this.emitter.emit('complete'); + this.dispose(); } } private runSync() { - for (const action of this.actions) { - //if the cancellation token has asked us to cancel, then stop processing now - if (this.options.cancellationToken.isCancellationRequested) { - return this.handleCancel(); + try { + for (const action of this.actions) { + //if the cancellation token has asked us to cancel, then stop processing now + if (this.options.cancellationToken.isCancellationRequested) { + return this.handleCancel(); + } + action.func(...action.args); } - action.func(...action.args); + } finally { + this.emitter.emit('complete'); + this.dispose(); } } @@ -82,4 +97,8 @@ export class Sequencer { console.log(`Cancelling sequence ${this.options.name}`); this.emitter.emit('cancel'); } + + private dispose() { + this.emitter.removeAllListeners(); + } } From 7d72c48383179abc4f1f643493ca81e1e37d6733 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Thu, 11 Jan 2024 17:00:25 -0500 Subject: [PATCH 018/119] Prevent crashing worker thread project --- src/lsp/worker/WorkerThreadProjectRunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lsp/worker/WorkerThreadProjectRunner.ts b/src/lsp/worker/WorkerThreadProjectRunner.ts index b2cbad1c3..a60b33275 100644 --- a/src/lsp/worker/WorkerThreadProjectRunner.ts +++ b/src/lsp/worker/WorkerThreadProjectRunner.ts @@ -47,7 +47,7 @@ export class WorkerThreadProjectRunner { */ private onActivate() { //clean up any existing project - this.project.dispose(); + this.project?.dispose(); //make a new instance of the project (which is the same way we run it in the main thread). this.project = new Project(); From 059c67e21857c22a15167b4707fc9e051ae384ff Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Fri, 12 Jan 2024 23:14:38 -0500 Subject: [PATCH 019/119] Support cancelling validation in the worker thread --- src/Program.ts | 2 +- src/common/Sequencer.ts | 7 +++++++ src/lsp/LspProject.ts | 7 ++++++- src/lsp/Project.ts | 19 +++++++++++++++--- src/lsp/ProjectManager.ts | 28 ++------------------------- src/lsp/worker/WorkerThreadProject.ts | 8 ++++++++ 6 files changed, 40 insertions(+), 31 deletions(-) diff --git a/src/Program.ts b/src/Program.ts index b43454295..0bb274d45 100644 --- a/src/Program.ts +++ b/src/Program.ts @@ -708,7 +708,7 @@ export class Program { .onCancel(() => { timeEnd('cancelled'); }) - .onComplete(() => { + .onSuccess(() => { timeEnd(); }) .run(); diff --git a/src/common/Sequencer.ts b/src/common/Sequencer.ts index 1e954fcf7..b74a4e632 100644 --- a/src/common/Sequencer.ts +++ b/src/common/Sequencer.ts @@ -41,6 +41,11 @@ export class Sequencer { return this; } + public onSuccess(callback: () => void) { + this.emitter.on('success', callback); + return this; + } + public once(func: () => any) { this.actions.push({ args: [], @@ -72,6 +77,7 @@ export class Sequencer { } action.func(...action.args); } + this.emitter.emit('success'); } finally { this.emitter.emit('complete'); this.dispose(); @@ -87,6 +93,7 @@ export class Sequencer { } action.func(...action.args); } + this.emitter.emit('success'); } finally { this.emitter.emit('complete'); this.dispose(); diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 6e6a1be39..33cd09a45 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -27,7 +27,12 @@ export interface LspProject { * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project */ - validate(options: { cancellationToken: CancellationToken }): Promise; + validate(): Promise; + + /** + * Cancel any active validation that's running + */ + cancelValidate(): Promise; /** * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index a7eda5d20..f12787e56 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -8,7 +8,7 @@ import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; -import type { CancellationToken } from 'vscode-languageserver-protocol'; +import { CancellationTokenSource } from 'vscode-languageserver-protocol'; export class Project implements LspProject { /** @@ -74,17 +74,30 @@ export class Project implements LspProject { this.isActivated.resolve(); } + private cancellationTokenForValidate: CancellationTokenSource; + /** * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project */ - public async validate(options: { cancellationToken: CancellationToken }) { + public async validate() { + this.cancelValidate(); + //store + this.cancellationTokenForValidate = new CancellationTokenSource(); + await this.builder.program.validate({ async: true, - ...options + cancellationToken: this.cancellationTokenForValidate.token }); } + /** + * Cancel any active validation that's running + */ + public async cancelValidate() { + this.cancellationTokenForValidate?.cancel(); + } + /** * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. */ diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 060c36ac0..16342c843 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -43,7 +43,7 @@ export class ProjectManager { //cancel any active validation that's going on console.log('cancelValidate'); - this.cancelValidateProject(project); + await project.cancelValidate(); //apply every file event to this project await Promise.all(event.actions.map(async (action) => { @@ -55,34 +55,10 @@ export class ProjectManager { })); //now that all the files have been sent, validate the project (which will trigger downstream diagnostics to flow) - await this.validateProject(project); + await project.validate(); })); } - private pendingValidations = new Cache(); - - /** - * Validate the given project. This wraps the call in locks to ensure other actions will wait - * @param project - */ - private async validateProject(project: LspProject) { - const cancellationToken = new CancellationTokenSource(); - this.pendingValidations.getOrAdd(project, () => []).push(cancellationToken); - await project.validate({ - cancellationToken: cancellationToken.token - }); - } - - private cancelValidateProject(project: LspProject) { - const tokens = this.pendingValidations.getOrAdd(project, () => []); - - //remove all the tokens from the list, and cancel each one of them - for (const token of tokens.splice(0, tokens.length)) { - token.cancel(); - } - } - - /** * Given a list of all desired projects, create any missing projects and destroy and projects that are no longer available * Treat workspaces that don't have a bsconfig.json as a project. diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 341421f4d..9984e2d1a 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -64,6 +64,14 @@ export class WorkerThreadProject implements LspProject { return response.data; } + /** + * Cancel any active validation that's running + */ + public async cancelValidate() { + const response = await this.messageHandler.sendRequest('cancelValidate'); + return response.data; + } + /** * A local copy of all the file paths loaded in this program. This needs to stay in sync with any files we add/delete in the worker thread, * so we can keep doing in-process `.hasFile()` checks From 083accf38dc8245610b81f3239b2776fd8dcdca5 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sat, 13 Jan 2024 08:23:32 -0500 Subject: [PATCH 020/119] Apply all file changes in single chunk --- src/LanguageServer.ts | 13 +++-- src/Program.spec.ts | 14 +++++ src/Program.ts | 14 +++-- src/common/Sequencer.ts | 10 +++- src/lsp/LspProject.ts | 16 +++--- src/lsp/Project.ts | 76 +++++++++++++++++++++------ src/lsp/ProjectManager.ts | 17 +----- src/lsp/worker/WorkerThreadProject.ts | 17 ++---- 8 files changed, 115 insertions(+), 62 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 288a7a2b2..7aba81d9a 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -590,7 +590,7 @@ export class LanguageServer implements OnHandler { * Ask the client for the list of `files.exclude` patterns. Useful when determining if we should process a file */ private async getWorkspaceExcludeGlobs(workspaceFolder: string): Promise { - const config = await this.getClientConfiguration(workspaceFolder, 'files'); + const config = await this.getClientConfiguration<{ exclude: string[] }>(workspaceFolder, 'files'); return Object .keys(config?.exclude ?? {}) .filter(x => config?.exclude?.[x]) @@ -621,13 +621,13 @@ export class LanguageServer implements OnHandler { let workspaces = await Promise.all( (await this.connection.workspace.getWorkspaceFolders() ?? []).map(async (x) => { const workspaceFolder = util.uriToPath(x.uri); - const config = await this.getClientConfiguration(x.uri, 'brightscript'); + const config = await this.getClientConfiguration(x.uri, 'brightscript'); return { workspaceFolder: workspaceFolder, excludePatterns: await this.getWorkspaceExcludeGlobs(workspaceFolder), bsconfigPath: config.configFile, //TODO we need to solidify the actual name of this flag in user/workspace settings - threadingEnabled: config.threadingEnabled + threadingEnabled: config.languageServer.enableThreading } as WorkspaceConfig; }) @@ -1078,3 +1078,10 @@ type Handler = { type OnHandler = { [K in keyof Handler]: Handler[K] extends (arg: infer U) => void ? U : never; }; + +interface BrightScriptClientConfiguration { + configFile: string; + languageServer: { + enableThreading: boolean; + } +} diff --git a/src/Program.spec.ts b/src/Program.spec.ts index feac43a99..283f51969 100644 --- a/src/Program.spec.ts +++ b/src/Program.spec.ts @@ -551,6 +551,20 @@ describe.only('Program', () => { program.validate(); expectZeroDiagnostics(program); }); + + it('properly handles errors in async mode', async () => { + const file = program.setFile('source/main.brs', ``); + file.validate = function () { + throw new Error('Crash for test'); + }; + let error: Error; + try { + await program.validate({ async: true }); + } catch (e) { + error = e as any; + } + expect(error?.message).to.eql('Crash for test'); + }); }); describe('hasFile', () => { diff --git a/src/Program.ts b/src/Program.ts index 0bb274d45..42e7cd53a 100644 --- a/src/Program.ts +++ b/src/Program.ts @@ -656,6 +656,11 @@ export class Program { } } + /** + * Counter used to track which validation run is being logged + */ + private validationRunSequence = 0; + /** * Traverse the entire project, and validate all scopes */ @@ -663,15 +668,18 @@ export class Program { public validate(options: { async: false; cancellationToken?: CancellationToken }): void; public validate(options: { async: true; cancellationToken?: CancellationToken }): Promise; public validate(options?: { async?: boolean; cancellationToken?: CancellationToken }) { - const timeEnd = this.logger.timeStart(LogLevel.log, 'Validating project'); + const timeEnd = this.logger.timeStart(LogLevel.log, `Validating project${this.logger.logLevel > LogLevel.log ? ` (run ${this.validationRunSequence++})` : ''}`); const sequencer = new Sequencer({ name: 'program.validate', async: options?.async ?? false, - cancellationToken: options?.cancellationToken ?? new CancellationTokenSource().token + cancellationToken: options?.cancellationToken ?? new CancellationTokenSource().token, + //how many milliseconds can pass while doing synchronous operations before we register a short timeout + minSyncDuration: 150 }); - //for every unvalidated file, validate it + //this sequencer allows us to run in both sync and async mode, depending on whether options.async is enabled. + //We use this to prevent starving the CPU during long validate cycles when running in a language server context return sequencer .once(() => { this.diagnostics = []; diff --git a/src/common/Sequencer.ts b/src/common/Sequencer.ts index b74a4e632..d43b1a121 100644 --- a/src/common/Sequencer.ts +++ b/src/common/Sequencer.ts @@ -11,6 +11,10 @@ export class Sequencer { name: string; async?: boolean; cancellationToken?: CancellationToken; + /** + * The number of operations to run before registering a nexttick + */ + minSyncDuration: number; } ) { @@ -67,9 +71,13 @@ export class Sequencer { private async runAsync() { try { + let start = Date.now(); for (const action of this.actions) { //register a very short timeout between every action so we don't hog the CPU - await util.sleep(1); + if (Date.now() - start > this.options.minSyncDuration) { + await util.sleep(1); + start = Date.now(); + } //if the cancellation token has asked us to cancel, then stop processing now if (this.options.cancellationToken?.isCancellationRequested) { diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 33cd09a45..f7f486dfb 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,6 +1,7 @@ import type { CancellationToken, Diagnostic } from 'vscode-languageserver'; import type { SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; +import { DocumentAction } from './DocumentManager'; /** * Defines the contract between the ProjectManager and the main or worker thread Project classes @@ -61,17 +62,12 @@ export interface LspProject { hasFile(srcPath: string): MaybePromise; /** - * Add or replace the in-memory contents of the file at the specified path. This is typically called as the user is typing. - * @param srcPath absolute path to the file - * @param fileContents the contents of the file + * Apply a series of file changes to the program. + * This will cancel any active validation. + * @param documentActions + * @returns a boolean indicating whether this project accepted any of the file changes. If false, then this project didn't recognize any of the files and thus did nothing */ - setFile(srcPath: string, fileContents: string): MaybePromise; - - /** - * Remove the in-memory file at the specified path. This is typically called when the user (or file system watcher) triggers a file delete - * @param srcPath absolute path to the file - */ - removeFile(srcPath: string): MaybePromise; + applyFileChanges(documentActions: DocumentAction[]): Promise /** * An event that is emitted anytime the diagnostics for the project have changed (typically after a validate cycle has finished) diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index f12787e56..4a8ded72c 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -9,6 +9,7 @@ import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; +import { DocumentAction } from './DocumentManager'; export class Project implements LspProject { /** @@ -74,20 +75,22 @@ export class Project implements LspProject { this.isActivated.resolve(); } - private cancellationTokenForValidate: CancellationTokenSource; + private validationCancelToken: CancellationTokenSource; /** * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, - * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project + * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project. + * + * This will cancel any currently running validation and then run a new one. */ public async validate() { this.cancelValidate(); //store - this.cancellationTokenForValidate = new CancellationTokenSource(); + this.validationCancelToken = new CancellationTokenSource(); await this.builder.program.validate({ async: true, - cancellationToken: this.cancellationTokenForValidate.token + cancellationToken: this.validationCancelToken.token }); } @@ -95,7 +98,8 @@ export class Project implements LspProject { * Cancel any active validation that's running */ public async cancelValidate() { - this.cancellationTokenForValidate?.cancel(); + this.validationCancelToken?.cancel(); + delete this.validationCancelToken; } /** @@ -138,28 +142,68 @@ export class Project implements LspProject { return this.builder.program.hasFile(srcPath); } + /** + * Add or replace the in-memory contents of the file at the specified path. This is typically called as the user is typing. + * This will cancel any pending validation cycles and queue a future validation cycle instead. + * @param srcPath absolute path to the file + * @param fileContents the contents of the file + */ + public async applyFileChanges(documentActions: DocumentAction[]): Promise { + let didChangeFiles = false; + for (const action of documentActions) { + if (this.hasFile(action.srcPath)) { + if (action.type === 'set') { + didChangeFiles ||= this.setFile(action.srcPath, action.fileContents); + } else if (action.type === 'delete') { + didChangeFiles ||= this.removeFile(action.srcPath); + } + } + } + if (didChangeFiles) { + this.validate(); + } + return didChangeFiles; + } + /** * Set new contents for a file. This is safe to call any time. Changes will be queued and flushed at the correct times * during the program's lifecycle flow * @param srcPath absolute source path of the file * @param fileContents the text contents of the file + * @returns true if this program accepted and added the file. false if this file doesn't match against the program's files array */ - public setFile(srcPath: string, fileContents: string) { - this.builder.program.setFile( - { - src: srcPath, - dest: rokuDeploy.getDestPath(srcPath, this.getFilePaths(), this.builder.program.options.rootDir) - }, - fileContents - ); + private setFile(srcPath: string, fileContents: string) { + const { files, rootDir } = this.builder.program.options; + + //get the dest path for this file. + let destPath = rokuDeploy.getDestPath(srcPath, files, rootDir); + + //if we got a dest path, then the program wants this file + if (destPath) { + this.builder.program.setFile( + { + src: srcPath, + dest: destPath + }, + fileContents + ); + return true; + } + return false; } /** * Remove the in-memory file at the specified path. This is typically called when the user (or file system watcher) triggers a file delete - * @param srcPath absolute path to the file + * @param srcPath absolute path to the File + * @returns true if we found and removed the file. false if we didn't have a file to remove */ - public removeFile(srcPath: string) { - this.builder.program.removeFile(srcPath); + private removeFile(srcPath: string) { + if (this.builder.program.hasFile(srcPath)) { + this.builder.program.removeFile(srcPath); + return true; + } else { + return false; + } } /** diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 16342c843..1eec73676 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -40,22 +40,7 @@ export class ProjectManager { private async applyDocumentChanges(event: FlushEvent) { //apply all of the document actions to each project in parallel await Promise.all(this.projects.map(async (project) => { - - //cancel any active validation that's going on - console.log('cancelValidate'); - await project.cancelValidate(); - - //apply every file event to this project - await Promise.all(event.actions.map(async (action) => { - if (action.type === 'set') { - await project.setFile(action.srcPath, action.fileContents); - } else if (action.type === 'delete') { - await project.removeFile(action.srcPath); - } - })); - - //now that all the files have been sent, validate the project (which will trigger downstream diagnostics to flow) - await project.validate(); + await project.applyFileChanges(event.actions); })); } diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 9984e2d1a..733f6509f 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -10,6 +10,7 @@ import { WorkerThreadProjectRunner } from './WorkerThreadProjectRunner'; import { WorkerPool } from './WorkerPool'; import type { SemanticToken } from '../../interfaces'; import type { BsConfig } from '../../BsConfig'; +import { DocumentAction } from '../DocumentManager'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -96,23 +97,13 @@ export class WorkerThreadProject implements LspProject { * @param srcPath absolute source path of the file * @param fileContents the text contents of the file */ - public async setFile(srcPath: string, fileContents: string) { - const response = await this.messageHandler.sendRequest('setFile', { - data: [srcPath, fileContents] + public async applyFileChanges(documentActions: DocumentAction[]) { + const response = await this.messageHandler.sendRequest('applyFileChanges', { + data: [documentActions] }); return response.data; } - /** - * Remove the in-memory file at the specified path. This is typically called when the user (or file system watcher) triggers a file delete - * @param srcPath absolute path to the file - */ - public async removeFile(srcPath: string) { - const response = await this.messageHandler.sendRequest('removeFile', { - data: [srcPath] - }); - return response.data; - } /** * Get the list of all file paths that are currently loaded in the project From e4aea1b9a77bedfdff9ea769da9b7d009df363a0 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sun, 14 Jan 2024 07:06:54 -0500 Subject: [PATCH 021/119] Speed up semantic tokens performance --- src/LanguageServer.ts | 15 ++++++++------ src/ProgramBuilder.ts | 29 ++++++++++++++++++--------- src/lsp/LspProject.ts | 5 +++++ src/lsp/Project.ts | 28 +++++++++++++++++++------- src/lsp/ProjectManager.ts | 5 +++-- src/lsp/worker/WorkerThreadProject.ts | 14 +++++++++++++ 6 files changed, 71 insertions(+), 25 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 7aba81d9a..25c85fffb 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -26,7 +26,8 @@ import type { InitializeResult, CompletionParams, ResultProgressReporter, - WorkDoneProgressReporter + WorkDoneProgressReporter, + SemanticTokensOptions } from 'vscode-languageserver/node'; import { SemanticTokensRequest, @@ -35,6 +36,8 @@ import { FileChangeType, ProposedFeatures, TextDocuments, + TextDocumentSyncKind, + CodeActionKind, } from 'vscode-languageserver/node'; import { URI } from 'vscode-uri'; import { TextDocument } from 'vscode-languageserver-textdocument'; @@ -165,7 +168,7 @@ export class LanguageServer implements OnHandler { //return the capabilities of the server return { capabilities: { - // textDocumentSync: TextDocumentSyncKind.Full, + textDocumentSync: TextDocumentSyncKind.Full, // // Tell the client that the server supports code completion // completionProvider: { // resolveProvider: true, @@ -175,10 +178,10 @@ export class LanguageServer implements OnHandler { // }, // documentSymbolProvider: true, // workspaceSymbolProvider: true, - // semanticTokensProvider: { - // legend: semanticTokensLegend, - // full: true - // } as SemanticTokensOptions + semanticTokensProvider: { + legend: semanticTokensLegend, + full: true + } as SemanticTokensOptions, // referencesProvider: true, // codeActionProvider: { // codeActionKinds: [CodeActionKind.Refactor] diff --git a/src/ProgramBuilder.ts b/src/ProgramBuilder.ts index 0862edc60..d9d776939 100644 --- a/src/ProgramBuilder.ts +++ b/src/ProgramBuilder.ts @@ -94,7 +94,7 @@ export class ProgramBuilder { ]; } - public async run(options: BsConfig) { + public async run(options: BsConfig & { skipInitialValidation?: boolean }) { this.logger.logLevel = options.logLevel as LogLevel; if (this.isRunning) { @@ -135,10 +135,14 @@ export class ProgramBuilder { if (this.options.watch) { this.logger.log('Starting compilation in watch mode...'); - await this.runOnce(); + await this.runOnce({ + skipValidation: options.skipInitialValidation + }); this.enableWatchMode(); } else { - await this.runOnce(); + await this.runOnce({ + skipValidation: options.skipInitialValidation + }); } } @@ -255,14 +259,17 @@ export class ProgramBuilder { /** * Run the entire process exactly one time. */ - private runOnce() { + private runOnce(options?: { skipValidation?: boolean; }) { //clear the console this.clearConsole(); let cancellationToken = { isCanceled: false }; //wait for the previous run to complete let runPromise = this.cancelLastRun().then(() => { //start the new run - return this._runOnce(cancellationToken); + return this._runOnce({ + cancellationToken: cancellationToken, + skipValidation: options?.skipValidation + }); }) as any; //a function used to cancel this run @@ -338,18 +345,20 @@ export class ProgramBuilder { * Run the process once, allowing cancelability. * NOTE: This should only be called by `runOnce`. */ - private async _runOnce(cancellationToken: { isCanceled: any }) { + private async _runOnce(options: { cancellationToken: { isCanceled: any }; skipValidation: boolean; }) { let wereDiagnosticsPrinted = false; try { //maybe cancel? - if (cancellationToken.isCanceled === true) { + if (options.cancellationToken.isCanceled === true) { return -1; } //validate program - this.validateProject(); + if (options.skipValidation !== true) { + this.validateProject(); + } //maybe cancel? - if (cancellationToken.isCanceled === true) { + if (options.cancellationToken.isCanceled === true) { return -1; } @@ -367,7 +376,7 @@ export class ProgramBuilder { await this.createPackageIfEnabled(); //maybe cancel? - if (cancellationToken.isCanceled === true) { + if (options.cancellationToken.isCanceled === true) { return -1; } diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index f7f486dfb..6aa041ca9 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -24,6 +24,11 @@ export interface LspProject { */ activate(options: ActivateOptions): MaybePromise; + /** + * Get a promise that resolves when the project finishes activating + */ + whenActivated(): Promise; + /** * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 4a8ded72c..185fcc025 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -12,11 +12,13 @@ import { CancellationTokenSource } from 'vscode-languageserver-protocol'; import { DocumentAction } from './DocumentManager'; export class Project implements LspProject { + + /** * Activates this project. Every call to `activate` should completely reset the project, clear all used ram and start from scratch. */ public async activate(options: ActivateOptions) { - this.isActivated = new Deferred(); + this.activationDeferred = new Deferred(); this.projectPath = options.projectPath; this.workspaceFolder = options.workspaceFolder; @@ -58,7 +60,8 @@ export class Project implements LspProject { createPackage: false, deploy: false, copyToStaging: false, - showDiagnosticsInConsole: false + showDiagnosticsInConsole: false, + skipInitialValidation: true }); //if we found a deprecated brsconfig.json, add a diagnostic warning the user @@ -69,10 +72,21 @@ export class Project implements LspProject { }); } + //trigger a validation (but don't wait for it. That way we can cancel it sooner if we get new incoming data or requests) + void this.validate(); + //flush any diagnostics generated by this initial run this.emit('diagnostics', { diagnostics: this.getDiagnostics() }); - this.isActivated.resolve(); + this.activationDeferred.resolve(); + } + + /** + * Promise that resolves when the project finishes activating + * @returns + */ + public whenActivated() { + return this.activationDeferred.promise; } private validationCancelToken: CancellationTokenSource; @@ -112,7 +126,7 @@ export class Project implements LspProject { /** * Gets resolved when the project has finished activating */ - private isActivated: Deferred; + private activationDeferred: Deferred; public getDiagnostics() { return this.builder.getDiagnostics().map(x => { @@ -129,7 +143,7 @@ export class Project implements LspProject { */ private async onIdle(): Promise { await Promise.all([ - this.isActivated.promise + this.activationDeferred.promise ]); } @@ -313,8 +327,8 @@ export class Project implements LspProject { public dispose() { this.builder?.dispose(); this.emitter.removeAllListeners(); - if (this.isActivated?.isCompleted === false) { - this.isActivated.reject( + if (this.activationDeferred?.isCompleted === false) { + this.activationDeferred.reject( new Error('Project was disposed, activation has been aborted') ); } diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 1eec73676..c87466134 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -6,11 +6,9 @@ import type { LspDiagnostic, LspProject, MaybePromise } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; import type { Position } from 'vscode-languageserver'; -import { CancellationTokenSource } from 'vscode-languageserver'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; -import { Cache } from '../Cache'; /** * Manages all brighterscript projects for the language server @@ -111,6 +109,9 @@ export class ProjectManager { let doneCount = 0; this.projects.map(async (project) => { try { + //wait for the project to activate + await project.whenActivated(); + if (await Promise.resolve(callback(project)) === true) { deferred.tryResolve(project); } diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 733f6509f..83b1c3007 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -11,6 +11,7 @@ import { WorkerPool } from './WorkerPool'; import type { SemanticToken } from '../../interfaces'; import type { BsConfig } from '../../BsConfig'; import { DocumentAction } from '../DocumentManager'; +import { Deferred } from '../../deferred'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -54,8 +55,21 @@ export class WorkerThreadProject implements LspProject { //populate a few properties with data from the thread so we can use them for some synchronous checks this.filePaths = await this.getFilePaths(); this.options = await this.getOptions(); + + this.activationDeferred.resolve(); } + private activationDeferred = new Deferred(); + + /** + * Promise that resolves when the project finishes activating + * @returns + */ + public whenActivated() { + return this.activationDeferred.promise; + } + + /** * Validate the project. This will trigger a full validation on any scopes that were changed since the last validation, * and will also eventually emit a new 'diagnostics' event that includes all diagnostics for the project From 49eb6e231a61f61f646c46fb6c0981bea47615c4 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sun, 14 Jan 2024 23:56:12 -0500 Subject: [PATCH 022/119] Fix semantic tokens in worker project. --- src/lsp/DocumentManager.ts | 11 +++++++++++ src/lsp/ProjectManager.ts | 5 ++++- src/lsp/worker/WorkerThreadProject.ts | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/lsp/DocumentManager.ts b/src/lsp/DocumentManager.ts index 280c76f86..343499b3d 100644 --- a/src/lsp/DocumentManager.ts +++ b/src/lsp/DocumentManager.ts @@ -64,6 +64,17 @@ export class DocumentManager { this.emitSync('flush', event); } + /** + * Returns a promise that resolves when there are no pending files. Will immediately resolve if there are no files, + * and will wait until files are flushed if there are files. + */ + public async onSettle() { + if (this.queue.size > 0) { + await this.once('flush'); + return this.onSettle(); + } + } + public once(eventName: 'flush'): Promise; public once(eventName: string): Promise { return new Promise((resolve) => { diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index c87466134..b1e5e597b 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -103,10 +103,13 @@ export class ProjectManager { /** * Return the first project where the async matcher returns true */ - private findFirstMatchingProject(callback: (project: LspProject) => boolean | PromiseLike) { + private async findFirstMatchingProject(callback: (project: LspProject) => boolean | PromiseLike) { const deferred = new Deferred(); let projectCount = this.projects.length; let doneCount = 0; + //wait for pending document changes to settle + await this.documentManager.onSettle(); + this.projects.map(async (project) => { try { //wait for the project to activate diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 83b1c3007..fd01401bd 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -147,7 +147,9 @@ export class WorkerThreadProject implements LspProject { * @param srcPath absolute path to the source file */ public async getSemanticTokens(srcPath: string) { - const response = await this.messageHandler.sendRequest('getSemanticTokens'); + const response = await this.messageHandler.sendRequest('getSemanticTokens', { + data: [srcPath] + }); return response.data; } From e6f317e21982cde0db94da1d47b6c2486a095ef1 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 15 Jan 2024 00:02:04 -0500 Subject: [PATCH 023/119] Add baseline transpile preview support --- src/LanguageServer.ts | 12 ++++++------ src/lsp/LspProject.ts | 7 +++++++ src/lsp/Project.ts | 5 +++++ src/lsp/ProjectManager.ts | 15 +++++++++++++++ src/lsp/worker/WorkerThreadProject.ts | 8 ++++++++ 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 25c85fffb..1be1a0861 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -191,11 +191,11 @@ export class LanguageServer implements OnHandler { // }, // definitionProvider: true, // hoverProvider: true, - // executeCommandProvider: { - // commands: [ - // CustomCommands.TranspileFile - // ] - // } + executeCommandProvider: { + commands: [ + CustomCommands.TranspileFile + ] + } } as ServerCapabilities }; } @@ -557,7 +557,7 @@ export class LanguageServer implements OnHandler { public async onExecuteCommand(params: ExecuteCommandParams) { await this.waitAllProjectFirstRuns(); if (params.command === CustomCommands.TranspileFile) { - const result = await this.transpileFile(params.arguments[0]); + const result = await this.projectManager.transpileFile(params.arguments[0]); //back-compat: include `pathAbsolute` property so older vscode versions still work (result as any).pathAbsolute = result.srcPath; return result; diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 6aa041ca9..42867b0a2 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -2,6 +2,7 @@ import type { CancellationToken, Diagnostic } from 'vscode-languageserver'; import type { SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; import { DocumentAction } from './DocumentManager'; +import { FileTranspileResult } from '../Program'; /** * Defines the contract between the ProjectManager and the main or worker thread Project classes @@ -61,6 +62,12 @@ export interface LspProject { */ getSemanticTokens(srcPath: string): MaybePromise; + /** + * Transpile the specified file + * @param srcPath + */ + transpileFile(srcPath: string): MaybePromise; + /** * Does this project have the specified file. Should only be called after `.activate()` has completed. */ diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 185fcc025..5c47a4c43 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -236,6 +236,11 @@ export class Project implements LspProject { return this.builder.program.getSemanticTokens(srcPath); } + public async transpileFile(srcPath: string) { + await this.onIdle(); + return this.builder.program.getTranspiledFileContents(srcPath); + } + /** * Manages the BrighterScript program. The main interface into the compiler/validator */ diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index b1e5e597b..3e4224b95 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -146,6 +146,21 @@ export class ProjectManager { } } + public async transpileFile(srcPath: string) { + //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. + const project = await this.findFirstMatchingProject((p) => { + return p.hasFile(srcPath); + }); + + //if we found a project + if (project) { + const result = await Promise.resolve( + project.transpileFile(srcPath) + ); + return result; + } + } + public async getCompletions(srcPath: string, position: Position) { // const completions = await Promise.all( // this.projects.map(x => x.getCompletions(srcPath, position)) diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index fd01401bd..388c459cc 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -12,6 +12,7 @@ import type { SemanticToken } from '../../interfaces'; import type { BsConfig } from '../../BsConfig'; import { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; +import { FileTranspileResult } from '../../Program'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -153,6 +154,13 @@ export class WorkerThreadProject implements LspProject { return response.data; } + public async transpileFile(srcPath: string) { + const response = await this.messageHandler.sendRequest('transpileFile', { + data: [srcPath] + }); + return response.data; + } + /** * Handles request/response/update messages from the worker thread */ From 86e8f51d74c65dda379e27b70ef4ec068447ff57 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Wed, 17 Jan 2024 08:55:52 -0500 Subject: [PATCH 024/119] Fix performance issues for transpile preview --- src/LanguageServer.ts | 13 +---------- src/Logger.ts | 3 ++- src/Program.spec.ts | 8 +++---- src/Program.ts | 38 ++++++++++++++++----------------- src/files/BrsFile.ts | 20 +++++++---------- src/lsp/DocumentManager.spec.ts | 2 +- 6 files changed, 35 insertions(+), 49 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 1be1a0861..c73296ee6 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -998,17 +998,6 @@ export class LanguageServer implements OnHandler { } private diagnosticCollection = new DiagnosticCollection(); - private async transpileFile(srcPath: string) { - //wait all program first runs - await this.waitAllProjectFirstRuns(); - //find the first project that has this file - for (let project of this.getProjects()) { - if (project.builder.program.hasFile(srcPath)) { - return project.builder.program.getTranspiledFileContents(srcPath); - } - } - } - private getProjects() { //TODO delete this because projectManager handles all this stuff now return []; @@ -1086,5 +1075,5 @@ interface BrightScriptClientConfiguration { configFile: string; languageServer: { enableThreading: boolean; - } + }; } diff --git a/src/Logger.ts b/src/Logger.ts index 4e714a117..13dd52d2b 100644 --- a/src/Logger.ts +++ b/src/Logger.ts @@ -141,6 +141,7 @@ export class Logger { this[logLevelString](...messages ?? [], `${status}. (${chalk.blue(stopwatch.getDurationText())})`); }; } + return noop; } /** @@ -193,7 +194,7 @@ export class Logger { } } -export function noop() { +export function noop(...args: []): any { } diff --git a/src/Program.spec.ts b/src/Program.spec.ts index 283f51969..fa0e731f3 100644 --- a/src/Program.spec.ts +++ b/src/Program.spec.ts @@ -24,7 +24,7 @@ import { tempDir, rootDir, stagingDir } from './testHelpers.spec'; let sinon = sinonImport.createSandbox(); -describe.only('Program', () => { +describe('Program', () => { let program: Program; beforeEach(() => { @@ -43,7 +43,7 @@ describe.only('Program', () => { program.dispose(); }); - it('Does not crazy for file not referenced by any other scope', async () => { + it('Does not crash for file not referenced by any other scope', async () => { program.setFile('tests/testFile.spec.bs', ` function main(args as object) as object return roca(args).describe("test suite", sub() @@ -1885,8 +1885,8 @@ describe.only('Program', () => { src: s`${rootDir}/source/main.bs`, dest: 'source/main.bs' }, { - src: s`${rootDir}/source/main.bs`, - dest: 'source/main.bs' + src: s`${rootDir}/source/common.bs`, + dest: 'source/common.bs' }], program.options.stagingDir); //entries should now be in alphabetic order diff --git a/src/Program.ts b/src/Program.ts index 42e7cd53a..6fd0460f7 100644 --- a/src/Program.ts +++ b/src/Program.ts @@ -1112,20 +1112,17 @@ export class Program { * @param filePath can be a srcPath or a destPath */ public async getTranspiledFileContents(filePath: string) { - let fileMap = await rokuDeploy.getFilePaths(this.options.files, this.options.rootDir); - //remove files currently loaded in the program, we will transpile those instead (even if just for source maps) - let filteredFileMap = [] as FileObj[]; - for (let fileEntry of fileMap) { - if (this.hasFile(fileEntry.src) === false) { - filteredFileMap.push(fileEntry); - } - } + const file = this.getFile(filePath); + const fileMap: FileObj[] = [{ + src: file.srcPath, + dest: file.pkgPath + }]; const { entries, astEditor } = this.beforeProgramTranspile(fileMap, this.options.stagingDir); const result = this._getTranspiledFileContents( - this.getFile(filePath) + file ); this.afterProgramTranspile(entries, astEditor); - return result; + return Promise.resolve(result); } /** @@ -1134,7 +1131,6 @@ export class Program { */ private _getTranspiledFileContents(file: BscFile, outputPath?: string): FileTranspileResult { const editor = new AstEditor(); - this.plugins.emit('beforeFileTranspile', { program: this, file: file, @@ -1204,15 +1200,19 @@ export class Program { return outputPath; }; - const entries = Object.values(this.files).map(file => { - return { - file: file, - outputPath: getOutputPath(file) - }; + const entries = Object.values(this.files) + //only include the files from fileEntries + .filter(file => !!mappedFileEntries[file.srcPath]) + .map(file => { + return { + file: file, + outputPath: getOutputPath(file) + }; + }) //sort the entries to make transpiling more deterministic - }).sort((a, b) => { - return a.file.srcPath < b.file.srcPath ? -1 : 1; - }); + .sort((a, b) => { + return a.file.srcPath < b.file.srcPath ? -1 : 1; + }); const astEditor = new AstEditor(); diff --git a/src/files/BrsFile.ts b/src/files/BrsFile.ts index 9edef7fd8..395361130 100644 --- a/src/files/BrsFile.ts +++ b/src/files/BrsFile.ts @@ -1232,7 +1232,7 @@ export class BrsFile { } /** - * Find the first scope that has a namespace with this name. + * Finds the first scope for this file, then returns true if there's a namespace with this name. * Returns false if no namespace was found with that name */ public calleeStartsWithNamespace(callee: Expression) { @@ -1244,11 +1244,9 @@ export class BrsFile { if (isVariableExpression(left)) { let lowerName = left.name.text.toLowerCase(); //find the first scope that contains this namespace - let scopes = this.program.getScopesForFile(this); - for (let scope of scopes) { - if (scope.namespaceLookup.has(lowerName)) { - return true; - } + let scope = this.program.getFirstScopeForFile(this); + if (scope?.namespaceLookup.has(lowerName)) { + return true; } } return false; @@ -1262,12 +1260,10 @@ export class BrsFile { if (isVariableExpression(callee) && namespaceName) { let lowerCalleeName = callee?.name?.text?.toLowerCase(); if (lowerCalleeName) { - let scopes = this.program.getScopesForFile(this); - for (let scope of scopes) { - let namespace = scope.namespaceLookup.get(namespaceName.toLowerCase()); - if (namespace.functionStatements[lowerCalleeName]) { - return true; - } + let scope = this.program.getFirstScopeForFile(this); + let namespace = scope.namespaceLookup.get(namespaceName.toLowerCase()); + if (namespace.functionStatements[lowerCalleeName]) { + return true; } } } diff --git a/src/lsp/DocumentManager.spec.ts b/src/lsp/DocumentManager.spec.ts index c3babaf6c..b8fa715c2 100644 --- a/src/lsp/DocumentManager.spec.ts +++ b/src/lsp/DocumentManager.spec.ts @@ -2,7 +2,7 @@ import { expect } from 'chai'; import util from '../util'; import { DocumentManager } from './DocumentManager'; -describe.only('DocumentManager', () => { +describe('DocumentManager', () => { let manager: DocumentManager; beforeEach(() => { manager = new DocumentManager({ From 21e8e7627df91b5e8406a25930cc75f6de25088a Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 18 Mar 2024 14:57:14 -0400 Subject: [PATCH 025/119] Add util.promiseRaceMatch --- src/util.spec.ts | 97 +++++++++++++++++++++++++++++++++++++++++++++++- src/util.ts | 55 +++++++++++++++++++++++++-- 2 files changed, 146 insertions(+), 6 deletions(-) diff --git a/src/util.spec.ts b/src/util.spec.ts index bc8f4faf5..80e907483 100644 --- a/src/util.spec.ts +++ b/src/util.spec.ts @@ -6,7 +6,7 @@ import type { BsConfig } from './BsConfig'; import * as fsExtra from 'fs-extra'; import { createSandbox } from 'sinon'; import { DiagnosticMessages } from './DiagnosticMessages'; -import { tempDir, rootDir } from './testHelpers.spec'; +import { tempDir, rootDir, expectThrows } from './testHelpers.spec'; import { Program } from './Program'; const sinon = createSandbox(); @@ -376,7 +376,7 @@ describe('util', () => { }); it('sets default value for bslibDestinationDir', () => { - expect(util.normalizeConfig({ }).bslibDestinationDir).to.equal('source'); + expect(util.normalizeConfig({}).bslibDestinationDir).to.equal('source'); }); it('strips leading and/or trailing slashes from bslibDestinationDir', () => { @@ -903,4 +903,97 @@ describe('util', () => { }]); }); }); + + describe.only('promiseRaceMatch', () => { + async function resolveAfter(value: T, timeout: number) { + await util.sleep(timeout); + return value; + } + + it('returns the value from the first promise that resolves that matches the matcher', async () => { + expect( + await util.promiseRaceMatch([ + resolveAfter('a', 1), + resolveAfter('b', 20), + resolveAfter('c', 30) + ], x => true) + ).to.eql('a'); + + expect( + await util.promiseRaceMatch([ + resolveAfter('a', 30), + resolveAfter('b', 1), + resolveAfter('c', 20) + ], x => true) + ).to.eql('b'); + + expect( + await util.promiseRaceMatch([ + resolveAfter('a', 20), + resolveAfter('b', 30), + resolveAfter('c', 1) + ], x => true) + ).to.eql('c'); + }); + + it('returns a value even if one of the promises never resolves', async () => { + expect( + await util.promiseRaceMatch([ + new Promise(() => { + //i will never resolve + }), + resolveAfter('a', 1) + ], x => true) + ).to.eql('a'); + }); + + it('rejects if all the promises fail', async () => { + let error: Error; + try { + await util.promiseRaceMatch([ + Promise.reject(new Error('error 1')), + Promise.reject(new Error('error 2')), + Promise.reject(new Error('error 3')) + ], x => true); + } catch (e) { + error = e as any; + } + expect( + (error as AggregateError).errors.map(x => x.message) + ).to.eql([ + 'error 1', + 'error 2', + 'error 3' + ]); + }); + + it('returns a value when one of the promises rejects', async () => { + expect( + await util.promiseRaceMatch([ + Promise.reject(new Error('crash')), + resolveAfter('a', 1) + ], x => true) + ).to.eql('a'); + }); + + it('returns undefined if no valuees match the matcher', async () => { + expect( + await util.promiseRaceMatch([ + resolveAfter('a', 1), + resolveAfter('b', 20), + resolveAfter('c', 30) + ], x => false) + ).to.be.undefined; + }); + + it('returns undefined if no matcher is provided', async () => { + expect( + await util.promiseRaceMatch([ + resolveAfter('a', 1), + resolveAfter('b', 20), + resolveAfter('c', 30) + ], undefined) + ).to.be.undefined; + }); + }); }); diff --git a/src/util.ts b/src/util.ts index 8690ed599..f062d2729 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1513,11 +1513,58 @@ export class Util { } /** - * Run a series of actions in chunks. This allows us to run them sequentially or each operation on nexttick to prevent starving the CPU - * @param options - */ - public runInChunks(options: { async: boolean; cancel: CancellationToken; actions: Array<{ collection: any[]; action: (item: any) => any }> }) { + * Race a series of promises, and return the first one that resolves AND matches the matcher function. + * If all of the promises reject, then this will emit an AggregatreError with all of the errors. + * If at least one promise resolves, then this will log all of the errors to the console + * If at least one promise resolves but none of them match the matcher, then this will return undefined. + * @param promises all of the promises to race + * @param matcher a function that should return true if this value should be kept. Returning any value other than true means `false` + * @returns the first resolved value that matches the matcher, or undefined if none of them match + */ + public async promiseRaceMatch(promises: Promise[], matcher: (value: T) => boolean) { + const workingPromises = [ + ...promises + ]; + + const results: Array<{ value: T; index: number } | { error: Error; index: number }> = []; + let returnValue: T; + + while (workingPromises.length > 0) { + //race the promises. If any of them resolve, evaluate it against the matcher. If that passes, return the value. otherwise, eliminate this promise and try again + const result = await Promise.race( + workingPromises.map((promise, i) => { + return promise + .then(value => ({ value: value, index: i })) + .catch(error => ({ error: error, index: i })); + }) + ); + results.push(result); + //if we got a value and it matches the matcher, return it + if ('value' in result && matcher?.(result.value) === true) { + returnValue = result.value; + break; + } + + //remove this non-matched (or errored) promise from the list and try again + workingPromises.splice(result.index, 1); + } + + const errors = (results as Array<{ error: Error }>) + .filter(x => 'error' in x) + .map(x => x.error); + + //if all of them crashed, then reject + if (errors.length === promises.length) { + throw new AggregateError(errors); + } else { + //log all of the errors + for (const error of errors) { + console.error(error); + } + } + //return the matched value, or undefined if there wasn't one + return returnValue; } } From 8d3c7c050430303520ce42d3cc2580bad314ab2f Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 18 Mar 2024 16:00:26 -0400 Subject: [PATCH 026/119] Add `hover` support --- src/LanguageServer.ts | 39 +++-------------------- src/Program.ts | 1 - src/interfaces.ts | 2 ++ src/lsp/LspProject.ts | 19 ++++++----- src/lsp/Project.ts | 19 ++++++----- src/lsp/ProjectManager.ts | 45 ++++++++++++++++++--------- src/lsp/worker/WorkerThreadProject.ts | 21 ++++++++----- src/util.ts | 8 ++--- 8 files changed, 77 insertions(+), 77 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index c73296ee6..e30c0180d 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -20,7 +20,6 @@ import type { SemanticTokens, SemanticTokensParams, TextDocumentChangeEvent, - Hover, HandlerResult, InitializeError, InitializeResult, @@ -36,8 +35,7 @@ import { FileChangeType, ProposedFeatures, TextDocuments, - TextDocumentSyncKind, - CodeActionKind, + TextDocumentSyncKind } from 'vscode-languageserver/node'; import { URI } from 'vscode-uri'; import { TextDocument } from 'vscode-languageserver-textdocument'; @@ -190,7 +188,7 @@ export class LanguageServer implements OnHandler { // triggerCharacters: ['(', ','] // }, // definitionProvider: true, - // hoverProvider: true, + hoverProvider: true, executeCommandProvider: { commands: [ CustomCommands.TranspileFile @@ -401,38 +399,9 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage public async onHover(params: TextDocumentPositionParams) { - //ensure programs are initialized - await this.waitAllProjectFirstRuns(); - const srcPath = util.uriToPath(params.textDocument.uri); - let projects = this.getProjects(); - let hovers = projects - //get hovers from all projects - .map((x) => x.builder.program.getHover(srcPath, params.position)) - //flatten to a single list - .flat(); - - const contents = [ - ...(hovers ?? []) - //pull all hover contents out into a flag array of strings - .map(x => { - return Array.isArray(x?.contents) ? x?.contents : [x?.contents]; - }).flat() - //remove nulls - .filter(x => !!x) - //dedupe hovers across all projects - .reduce((set, content) => set.add(content), new Set()).values() - ]; - - if (contents.length > 0) { - let hover: Hover = { - //use the range from the first hover - range: hovers[0]?.range, - //the contents of all hovers - contents: contents - }; - return hover; - } + const result = await this.projectManager.getHover(srcPath, params.position); + return result; } @AddStackToErrorMessage diff --git a/src/Program.ts b/src/Program.ts index 6fd0460f7..be1f7b7fb 100644 --- a/src/Program.ts +++ b/src/Program.ts @@ -24,7 +24,6 @@ import type { FunctionStatement, NamespaceStatement } from './parser/Statement'; import { BscPlugin } from './bscPlugin/BscPlugin'; import { AstEditor } from './astUtils/AstEditor'; import type { SourceMapGenerator } from 'source-map'; -import { rokuDeploy } from 'roku-deploy'; import type { Statement } from './parser/AstNode'; import { CallExpressionInfo } from './bscPlugin/CallExpressionInfo'; import { SignatureHelpUtil } from './bscPlugin/SignatureHelpUtil'; diff --git a/src/interfaces.ts b/src/interfaces.ts index 248d9e128..65d5b60e8 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -413,3 +413,5 @@ export interface FileLink { } export type DisposableLike = Disposable | (() => any); + +export type MaybePromise = T | Promise; diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 42867b0a2..145040869 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,8 +1,8 @@ -import type { CancellationToken, Diagnostic } from 'vscode-languageserver'; -import type { SemanticToken } from '../interfaces'; +import type { Diagnostic, Position } from 'vscode-languageserver'; +import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; -import { DocumentAction } from './DocumentManager'; -import { FileTranspileResult } from '../Program'; +import type { DocumentAction } from './DocumentManager'; +import type { FileTranspileResult } from '../Program'; /** * Defines the contract between the ProjectManager and the main or worker thread Project classes @@ -39,7 +39,7 @@ export interface LspProject { /** * Cancel any active validation that's running */ - cancelValidate(): Promise; + cancelValidate(): MaybePromise; /** * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. @@ -68,6 +68,11 @@ export interface LspProject { */ transpileFile(srcPath: string): MaybePromise; + /** + * Get the hover information for the specified position in the specified file + */ + getHover(options: { srcPath: string; position: Position }): MaybePromise; + /** * Does this project have the specified file. Should only be called after `.activate()` has completed. */ @@ -79,7 +84,7 @@ export interface LspProject { * @param documentActions * @returns a boolean indicating whether this project accepted any of the file changes. If false, then this project didn't recognize any of the files and thus did nothing */ - applyFileChanges(documentActions: DocumentAction[]): Promise + applyFileChanges(documentActions: DocumentAction[]): Promise; /** * An event that is emitted anytime the diagnostics for the project have changed (typically after a validate cycle has finished) @@ -117,5 +122,3 @@ export interface ActivateOptions { export interface LspDiagnostic extends Diagnostic { uri: string; } - -export type MaybePromise = T | Promise; diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 5c47a4c43..92303cc39 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -2,18 +2,18 @@ import { ProgramBuilder } from '../ProgramBuilder'; import * as EventEmitter from 'eventemitter3'; import util, { standardizePath as s } from '../util'; import * as path from 'path'; -import type { ActivateOptions, LspDiagnostic, LspProject, MaybePromise } from './LspProject'; -import type { CompilerPlugin } from '../interfaces'; +import type { ActivateOptions, LspDiagnostic, LspProject } from './LspProject'; +import type { CompilerPlugin, Hover, MaybePromise } from '../interfaces'; import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; +import type { Position } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; -import { DocumentAction } from './DocumentManager'; +import type { DocumentAction } from './DocumentManager'; export class Project implements LspProject { - /** * Activates this project. Every call to `activate` should completely reset the project, clear all used ram and start from scratch. */ @@ -83,7 +83,7 @@ export class Project implements LspProject { /** * Promise that resolves when the project finishes activating - * @returns + * @returns a promise that resolves when the project finishes activating */ public whenActivated() { return this.activationDeferred.promise; @@ -111,7 +111,7 @@ export class Project implements LspProject { /** * Cancel any active validation that's running */ - public async cancelValidate() { + public cancelValidate() { this.validationCancelToken?.cancel(); delete this.validationCancelToken; } @@ -174,7 +174,7 @@ export class Project implements LspProject { } } if (didChangeFiles) { - this.validate(); + await this.validate(); } return didChangeFiles; } @@ -241,6 +241,11 @@ export class Project implements LspProject { return this.builder.program.getTranspiledFileContents(srcPath); } + public async getHover(options: { srcPath: string; position: Position }): Promise { + await this.onIdle(); + return this.builder.program.getHover(options.srcPath, options.position); + } + /** * Manages the BrighterScript program. The main interface into the compiler/validator */ diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 3e4224b95..d5299060f 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -2,13 +2,14 @@ import { standardizePath as s, util } from '../util'; import { rokuDeploy } from 'roku-deploy'; import * as path from 'path'; import * as EventEmitter from 'eventemitter3'; -import type { LspDiagnostic, LspProject, MaybePromise } from './LspProject'; +import type { LspDiagnostic, LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import type { Position } from 'vscode-languageserver'; +import { type Hover, type Position } from 'vscode-languageserver'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; +import type { MaybePromise } from '../interfaces'; /** * Manages all brighterscript projects for the language server @@ -103,7 +104,7 @@ export class ProjectManager { /** * Return the first project where the async matcher returns true */ - private async findFirstMatchingProject(callback: (project: LspProject) => boolean | PromiseLike) { + private async findFirstMatchingProject(matcher: (project: LspProject) => boolean | PromiseLike) { const deferred = new Deferred(); let projectCount = this.projects.length; let doneCount = 0; @@ -115,7 +116,7 @@ export class ProjectManager { //wait for the project to activate await project.whenActivated(); - if (await Promise.resolve(callback(project)) === true) { + if (await Promise.resolve(matcher(project)) === true) { deferred.tryResolve(project); } } catch (e) { @@ -133,8 +134,8 @@ export class ProjectManager { public async getSemanticTokens(srcPath: string) { //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. - const project = await this.findFirstMatchingProject((p) => { - return p.hasFile(srcPath); + const project = await this.findFirstMatchingProject((x) => { + return x.hasFile(srcPath); }); //if we found a project @@ -147,7 +148,7 @@ export class ProjectManager { } public async transpileFile(srcPath: string) { - //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. + //find the first program that has this file const project = await this.findFirstMatchingProject((p) => { return p.hasFile(srcPath); }); @@ -171,6 +172,21 @@ export class ProjectManager { // } } + /** + * Get the hover information for the given position in the file. If multiple projects have hover information, the projects will be raced and + * the fastest result will be returned + * @returns the hover information or undefined if no hover information was found + */ + public async getHover(srcPath: string, position: Position): Promise { + //Ask every project for hover info, keep whichever one responds first that has a valid response + let hover = await util.promiseRaceMatch( + this.projects.map(x => x.getHover({ srcPath: srcPath, position: position })), + //keep the first non-falsey result + (result) => !!result + ); + return hover?.[0]; + } + /** * Scan a given workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. * If none are found, then the workspaceFolder itself is treated as a project @@ -254,16 +270,15 @@ export class ProjectManager { /** * Create a project for the given config - * @param config * @returns a new project, or the existing project if one already exists with this config info */ - private async createProject(config1: ProjectConfig) { + private async createProject(config: ProjectConfig) { //skip this project if we already have it - if (this.hasProject(config1.projectPath)) { - return this.getProject(config1.projectPath); + if (this.hasProject(config.projectPath)) { + return this.getProject(config.projectPath); } - let project: LspProject = config1.threadingEnabled + let project: LspProject = config.threadingEnabled ? new WorkerThreadProject() : new Project(); @@ -278,9 +293,9 @@ export class ProjectManager { }); await project.activate({ - projectPath: config1.projectPath, - workspaceFolder: config1.workspaceFolder, - projectNumber: config1.projectNumber ?? ProjectManager.projectNumberSequence++ + projectPath: config.projectPath, + workspaceFolder: config.workspaceFolder, + projectNumber: config.projectNumber ?? ProjectManager.projectNumberSequence++ }); console.log('Activated'); } diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 388c459cc..620842ad2 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -3,16 +3,17 @@ import { Worker } from 'worker_threads'; import type { WorkerMessage } from './MessageHandler'; import { MessageHandler } from './MessageHandler'; import util from '../../util'; -import type { LspDiagnostic, MaybePromise } from '../LspProject'; +import type { LspDiagnostic } from '../LspProject'; import { type ActivateOptions, type LspProject } from '../LspProject'; import { isMainThread, parentPort } from 'worker_threads'; import { WorkerThreadProjectRunner } from './WorkerThreadProjectRunner'; import { WorkerPool } from './WorkerPool'; -import type { SemanticToken } from '../../interfaces'; +import type { Hover, MaybePromise, SemanticToken } from '../../interfaces'; import type { BsConfig } from '../../BsConfig'; -import { DocumentAction } from '../DocumentManager'; +import type { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; -import { FileTranspileResult } from '../../Program'; +import type { FileTranspileResult } from '../../Program'; +import type { Position } from 'vscode-languageserver-protocol'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -64,7 +65,7 @@ export class WorkerThreadProject implements LspProject { /** * Promise that resolves when the project finishes activating - * @returns + * @returns a promise that resolves when the project finishes activating */ public whenActivated() { return this.activationDeferred.promise; @@ -109,8 +110,7 @@ export class WorkerThreadProject implements LspProject { /** * Set new contents for a file. This is safe to call any time. Changes will be queued and flushed at the correct times * during the program's lifecycle flow - * @param srcPath absolute source path of the file - * @param fileContents the text contents of the file + * @param documentActions absolute source path of the file */ public async applyFileChanges(documentActions: DocumentAction[]) { const response = await this.messageHandler.sendRequest('applyFileChanges', { @@ -161,6 +161,13 @@ export class WorkerThreadProject implements LspProject { return response.data; } + public async getHover(options: { srcPath: string; position: Position }): Promise { + const response = await this.messageHandler.sendRequest('getHover', { + data: [options] + }); + return response.data; + } + /** * Handles request/response/update messages from the worker thread */ diff --git a/src/util.ts b/src/util.ts index f062d2729..94f3bad0b 100644 --- a/src/util.ts +++ b/src/util.ts @@ -4,12 +4,12 @@ import type { ParseError } from 'jsonc-parser'; import { parse as parseJsonc, printParseErrorCode } from 'jsonc-parser'; import * as path from 'path'; import { rokuDeploy, DefaultFiles, standardizePath as rokuDeployStandardizePath } from 'roku-deploy'; -import type { Diagnostic, Position, Range, Location, CancellationToken } from 'vscode-languageserver'; +import type { Diagnostic, Position, Range, Location } from 'vscode-languageserver'; import { URI } from 'vscode-uri'; import * as xml2js from 'xml2js'; import type { BsConfig } from './BsConfig'; import { DiagnosticMessages } from './DiagnosticMessages'; -import type { CallableContainer, BsDiagnostic, FileReference, CallableContainerMap, CompilerPluginFactory, CompilerPlugin, ExpressionInfo, DisposableLike } from './interfaces'; +import type { CallableContainer, BsDiagnostic, FileReference, CallableContainerMap, CompilerPluginFactory, CompilerPlugin, ExpressionInfo, DisposableLike, MaybePromise } from './interfaces'; import { BooleanType } from './types/BooleanType'; import { DoubleType } from './types/DoubleType'; import { DynamicType } from './types/DynamicType'; @@ -1521,7 +1521,7 @@ export class Util { * @param matcher a function that should return true if this value should be kept. Returning any value other than true means `false` * @returns the first resolved value that matches the matcher, or undefined if none of them match */ - public async promiseRaceMatch(promises: Promise[], matcher: (value: T) => boolean) { + public async promiseRaceMatch(promises: MaybePromise[], matcher: (value: T) => boolean) { const workingPromises = [ ...promises ]; @@ -1533,7 +1533,7 @@ export class Util { //race the promises. If any of them resolve, evaluate it against the matcher. If that passes, return the value. otherwise, eliminate this promise and try again const result = await Promise.race( workingPromises.map((promise, i) => { - return promise + return Promise.resolve(promise) .then(value => ({ value: value, index: i })) .catch(error => ({ error: error, index: i })); }) From 3afd0144c66808778a0c6b8b4644c246ca69728e Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 18 Mar 2024 16:16:23 -0400 Subject: [PATCH 027/119] fix crash --- package.json | 2 +- src/common/Sequencer.spec.ts | 3 ++- src/util.ts | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 6e3f7e704..d8ee0f77a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "brighterscript", - "version": "0.65.26", + "version": "0.65.26-local", "description": "A superset of Roku's BrightScript language.", "scripts": { "preversion": "npm run build && npm run lint && npm run test", diff --git a/src/common/Sequencer.spec.ts b/src/common/Sequencer.spec.ts index fac02634c..3915e6add 100644 --- a/src/common/Sequencer.spec.ts +++ b/src/common/Sequencer.spec.ts @@ -8,7 +8,8 @@ describe('Sequencer', () => { const values = []; void new Sequencer({ name: 'test', - cancellationToken: cancellationTokenSource.token + cancellationToken: cancellationTokenSource.token, + minSyncDuration: 100 }).forEach([1, 2, 3], (i) => { values.push(i); if (i === 2) { diff --git a/src/util.ts b/src/util.ts index 394539aaf..dea2847d4 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1344,7 +1344,7 @@ export class Util { */ public sortByRange(locatables: T[]) { //sort the tokens by range - return locatables.sort((a, b) => { + return locatables?.sort((a, b) => { //start line if (a.range.start.line < b.range.start.line) { return -1; From 8369717c30bb4df632d7d45b442669a3231b54f6 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 18 Mar 2024 22:16:44 -0400 Subject: [PATCH 028/119] Add getDefinition, and fix a few case-sensitive quirks --- ...al3brighterscriptdistSemanticTokenUtils.js | 0 src/LanguageServer.ts | 13 ++--- src/SemanticTokenUtils.ts | 2 +- src/lsp/LspProject.ts | 8 +++- src/lsp/Project.ts | 16 ++++++- src/lsp/ProjectManager.ts | 20 +++++++- src/lsp/worker/WorkerThreadProject.ts | 47 +++++++++++-------- 7 files changed, 72 insertions(+), 34 deletions(-) create mode 100644 projectsrokulocal3brighterscriptdistSemanticTokenUtils.js diff --git a/projectsrokulocal3brighterscriptdistSemanticTokenUtils.js b/projectsrokulocal3brighterscriptdistSemanticTokenUtils.js new file mode 100644 index 000000000..e69de29bb diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 9e7ab7ae3..a0a25474f 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -187,7 +187,7 @@ export class LanguageServer implements OnHandler { // signatureHelpProvider: { // triggerCharacters: ['(', ','] // }, - // definitionProvider: true, + definitionProvider: true, hoverProvider: true, executeCommandProvider: { commands: [ @@ -444,17 +444,10 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage @TrackBusyStatus public async onDefinition(params: TextDocumentPositionParams) { - await this.waitAllProjectFirstRuns(); - const srcPath = util.uriToPath(params.textDocument.uri); - const results = util.flatMap( - await Promise.all(this.getProjects().map(project => { - return project.builder.program.getDefinition(srcPath, params.position); - })), - c => c - ); - return results; + const result = this.projectManager.getDefinition(srcPath, params.position); + return result; } @AddStackToErrorMessage diff --git a/src/SemanticTokenUtils.ts b/src/SemanticTokenUtils.ts index a382fec9f..16d92256e 100644 --- a/src/SemanticTokenUtils.ts +++ b/src/SemanticTokenUtils.ts @@ -58,7 +58,7 @@ export const semanticTokensLegend = { export function encodeSemanticTokens(tokens: SemanticToken[]) { util.sortByRange(tokens); const builder = new SemanticTokensBuilder(); - for (const token of tokens) { + for (const token of tokens ?? []) { builder.push( token.range.start.line, token.range.start.character, diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 145040869..2d6cdc969 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,4 +1,4 @@ -import type { Diagnostic, Position } from 'vscode-languageserver'; +import type { Diagnostic, Position, Location } from 'vscode-languageserver'; import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; import type { DocumentAction } from './DocumentManager'; @@ -73,6 +73,12 @@ export interface LspProject { */ getHover(options: { srcPath: string; position: Position }): MaybePromise; + /** + * Get the locations where the symbol at the specified position is defined + * @param options the file path and position to get the definition for + */ + getDefinition(options: { srcPath: string; position: Position }): MaybePromise; + /** * Does this project have the specified file. Should only be called after `.activate()` has completed. */ diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 92303cc39..51ec0a3c8 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -8,7 +8,7 @@ import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; -import type { Position } from 'vscode-languageserver-protocol'; +import type { Location, Position } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; import type { DocumentAction } from './DocumentManager'; @@ -224,7 +224,14 @@ export class Project implements LspProject { * Get the list of all file paths that are currently loaded in the project */ public getFilePaths() { - return Object.keys(this.builder.program.files).sort(); + //get all the files in the program + return Object.values(this.builder.program.files) + //grab their srcPath values, and toLowerCase them here in case we're in a different thread just to save cycles from the main thread + .map(x => x.srcPath?.toLowerCase()) + //exclude nulls + .filter(x => !!x) + //sort them so it's easier to reason about downstream + .sort(); } /** @@ -246,6 +253,11 @@ export class Project implements LspProject { return this.builder.program.getHover(options.srcPath, options.position); } + public async getDefinition(options: { srcPath: string; position: Position }): Promise { + await this.onIdle(); + return this.builder.program.getDefinition(options.srcPath, options.position); + } + /** * Manages the BrighterScript program. The main interface into the compiler/validator */ diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index d5299060f..4f80d1ce5 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -5,7 +5,7 @@ import * as EventEmitter from 'eventemitter3'; import type { LspDiagnostic, LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import { type Hover, type Position } from 'vscode-languageserver'; +import type { Hover, Position, Location } from 'vscode-languageserver-protocol'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; @@ -187,6 +187,24 @@ export class ProjectManager { return hover?.[0]; } + /** + * Get the definition for the symbol at the given position in the file + * @param srcPath the path to the file + * @param position the position of symbol + * @returns a list of locations where the symbol under the position is defined in the project + */ + public async getDefinition(srcPath: string, position: Position): Promise { + //TODO should we merge definitions across ALL projects? or just return definitions from the first project we found + + //Ask every project for definition info, keep whichever one responds first that has a valid response + let result = await util.promiseRaceMatch( + this.projects.map(x => x.getDefinition({ srcPath: srcPath, position: position })), + //keep the first non-falsey result + (result) => !!result + ); + return result; + } + /** * Scan a given workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. * If none are found, then the workspaceFolder itself is treated as a project diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 620842ad2..b818859d6 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -13,7 +13,7 @@ import type { BsConfig } from '../../BsConfig'; import type { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; import type { FileTranspileResult } from '../../Program'; -import type { Position } from 'vscode-languageserver-protocol'; +import type { Position, Location } from 'vscode-languageserver-protocol'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -55,7 +55,7 @@ export class WorkerThreadProject implements LspProject { await this.messageHandler.sendRequest('activate', { data: [options] }); //populate a few properties with data from the thread so we can use them for some synchronous checks - this.filePaths = await this.getFilePaths(); + this.filePaths = new Set(await this.getFilePaths()); this.options = await this.getOptions(); this.activationDeferred.resolve(); @@ -90,10 +90,10 @@ export class WorkerThreadProject implements LspProject { } /** - * A local copy of all the file paths loaded in this program. This needs to stay in sync with any files we add/delete in the worker thread, - * so we can keep doing in-process `.hasFile()` checks + * A local copy of all the file paths loaded in this program, stored in lower case. + * This needs to stay in sync with any files we add/delete in the worker thread so we can keep doing in-process `.hasFile()` checks. */ - private filePaths: string[]; + private filePaths: Set; public async getDiagnostics() { const response = await this.messageHandler.sendRequest('getDiagnostics'); @@ -101,10 +101,10 @@ export class WorkerThreadProject implements LspProject { } /** - * Does this project have the specified file. Should only be called after `.activate()` has finished/ + * Does this project have the specified file. Should only be called after `.activate()` has finished. */ public hasFile(srcPath: string) { - return this.filePaths.includes(srcPath); + return this.filePaths.has(srcPath.toLowerCase()); } /** @@ -143,31 +143,40 @@ export class WorkerThreadProject implements LspProject { return this.options.rootDir; } + /** + * Send a request with the standard structure + * @param name the name of the request + * @param data the array of data to send + * @returns the response from the request + */ + private async sendStandardRequest(name: string, ...data: any[]) { + const response = await this.messageHandler.sendRequest(name as any, { + data: data + }); + return response.data; + } + /** * Get the full list of semantic tokens for the given file path * @param srcPath absolute path to the source file */ public async getSemanticTokens(srcPath: string) { - const response = await this.messageHandler.sendRequest('getSemanticTokens', { - data: [srcPath] - }); - return response.data; + return this.sendStandardRequest('getSemanticTokens', srcPath); } public async transpileFile(srcPath: string) { - const response = await this.messageHandler.sendRequest('transpileFile', { - data: [srcPath] - }); - return response.data; + return this.sendStandardRequest('transpileFile', srcPath); } public async getHover(options: { srcPath: string; position: Position }): Promise { - const response = await this.messageHandler.sendRequest('getHover', { - data: [options] - }); - return response.data; + return this.sendStandardRequest('getHover', options); } + public async getDefinition(options: { srcPath: string; position: Position }): Promise { + return this.sendStandardRequest('getDefinition', options); + } + + /** * Handles request/response/update messages from the worker thread */ From 7d04deb3b0495cc0a08973526d978448013f0100 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 18 Mar 2024 22:32:10 -0400 Subject: [PATCH 029/119] Fix the busy status spinner --- src/LanguageServer.ts | 47 ++++++----------------------------- src/lsp/ProjectManager.ts | 52 +++++++++++++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 47 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index a0a25474f..c9ceac2f7 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -47,8 +47,6 @@ import { Throttler } from './Throttler'; import { DiagnosticCollection } from './DiagnosticCollection'; import { isBrsFile } from './astUtils/reflection'; import { encodeSemanticTokens, semanticTokensLegend } from './SemanticTokenUtils'; -import type { BusyStatus } from './BusyStatusTracker'; -import { BusyStatusTracker } from './BusyStatusTracker'; import type { WorkspaceConfig } from './lsp/ProjectManager'; import { ProjectManager } from './lsp/ProjectManager'; import type { LspDiagnostic, LspProject } from './lsp/LspProject'; @@ -97,8 +95,6 @@ export class LanguageServer implements OnHandler { return this.validateThrottler.run(this.boundValidateAll); } - public busyStatusTracker = new BusyStatusTracker(); - //run the server public run() { this.projectManager = new ProjectManager(); @@ -108,6 +104,10 @@ export class LanguageServer implements OnHandler { void this.sendDiagnostics(event); }); + this.projectManager.busyStatusTracker.on('change', (event) => { + this.sendBusyStatus(); + }); + //allow the lsp to provide file contents //TODO handle this... // this.projectManager.addFileResolver(this.documentFileResolver.bind(this)); @@ -115,11 +115,6 @@ export class LanguageServer implements OnHandler { // Create a connection for the server. The connection uses Node's IPC as a transport. this.establishConnection(); - // Send the current status of the busyStatusTracker anytime it changes - this.busyStatusTracker.on('change', (status) => { - this.sendBusyStatus(status); - }); - //listen to all of the output log events and pipe them into the debug channel in the extension this.loggerSubscription = Logger.subscribe((text) => { this.connection.tracer.log(text); @@ -202,7 +197,6 @@ export class LanguageServer implements OnHandler { * Called when the client has finished initializing */ @AddStackToErrorMessage - @TrackBusyStatus public async onInitialized() { try { if (this.hasConfigurationCapability) { @@ -235,7 +229,6 @@ export class LanguageServer implements OnHandler { * Provide a list of completion items based on the current cursor position */ @AddStackToErrorMessage - @TrackBusyStatus public async onCompletion1(params: CompletionParams, workDoneProgress: WorkDoneProgressReporter, resultProgress?: ResultProgressReporter) { const completions = await this.projectManager.getCompletions( util.uriToPath(params.textDocument.uri), @@ -261,7 +254,6 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - @TrackBusyStatus public async onCodeAction(params: CodeActionParams) { //ensure programs are initialized await this.waitAllProjectFirstRuns(); @@ -307,7 +299,6 @@ export class LanguageServer implements OnHandler { * file types are watched (.brs,.bs,.xml,manifest, and any json/text/image files) */ @AddStackToErrorMessage - @TrackBusyStatus private async onDidChangeWatchedFiles(params: DidChangeWatchedFilesParams) { //ensure programs are initialized await this.waitAllProjectFirstRuns(); @@ -405,7 +396,6 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - @TrackBusyStatus public async onWorkspaceSymbol(params: WorkspaceSymbolParams) { await this.waitAllProjectFirstRuns(); @@ -426,7 +416,6 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - @TrackBusyStatus public async onDocumentSymbol(params: DocumentSymbolParams) { await this.waitAllProjectFirstRuns(); @@ -442,7 +431,6 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - @TrackBusyStatus public async onDefinition(params: TextDocumentPositionParams) { const srcPath = util.uriToPath(params.textDocument.uri); @@ -451,7 +439,6 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - @TrackBusyStatus public async onSignatureHelp(params: SignatureHelpParams) { await this.waitAllProjectFirstRuns(); @@ -486,7 +473,6 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - @TrackBusyStatus public async onReferences(params: ReferenceParams) { await this.waitAllProjectFirstRuns(); @@ -504,7 +490,6 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage - @TrackBusyStatus private async onFullSemanticTokens(params: SemanticTokensParams) { const srcPath = util.uriToPath(params.textDocument.uri); const result = await this.projectManager.getSemanticTokens(srcPath); @@ -515,7 +500,6 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - @TrackBusyStatus public async onExecuteCommand(params: ExecuteCommandParams) { await this.waitAllProjectFirstRuns(); if (params.command === CustomCommands.TranspileFile) { @@ -539,14 +523,14 @@ export class LanguageServer implements OnHandler { * Send a new busy status notification to the client based on the current busy status * @param status */ - private sendBusyStatus(status: BusyStatus) { + private sendBusyStatus() { this.busyStatusIndex = ++this.busyStatusIndex <= 0 ? 0 : this.busyStatusIndex; void this.connection.sendNotification(NotificationName.busyStatus, { - status: status, + status: this.projectManager.busyStatusTracker.status, timestamp: Date.now(), index: this.busyStatusIndex, - activeRuns: [...this.busyStatusTracker.activeRuns] + activeRuns: [...this.projectManager.busyStatusTracker.activeRuns] }); } private busyStatusIndex = -1; @@ -580,7 +564,6 @@ export class LanguageServer implements OnHandler { * Handle situations where bsconfig.json files were added or removed (to elevate/lower workspaceFolder projects accordingly) * Leave existing projects alone if they are not affected by these changes */ - @TrackBusyStatus private async syncProjects() { // get all workspace paths from the client let workspaces = await Promise.all( @@ -905,14 +888,12 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - @TrackBusyStatus private onTextDocumentDidChangeContent(event: TextDocumentChangeEvent) { const srcPath = URI.parse(event.document.uri).fsPath; console.log('setFile', srcPath); this.projectManager.setFile(srcPath, event.document.getText()); } - @TrackBusyStatus private async validateAll() { try { //synchronize parsing for open files that were included/excluded from projects @@ -1010,20 +991,6 @@ function AddStackToErrorMessage(target: any, propertyKey: string, descriptor: Pr }; } -/** - * An annotation used to wrap the method in a busyStatus tracking call - */ -function TrackBusyStatus(target: any, propertyKey: string, descriptor: PropertyDescriptor) { - let originalMethod = descriptor.value; - - //wrapping the original method - descriptor.value = function value(this: LanguageServer, ...args: any[]) { - return this.busyStatusTracker.run(() => { - return originalMethod.apply(this, args); - }, originalMethod.name); - }; -} - type Handler = { [K in keyof T as K extends `on${string}` ? K : never]: T[K] extends (arg: infer U) => void ? (arg: U) => void : never; diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 4f80d1ce5..bd5fcacba 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -10,11 +10,17 @@ import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; import type { MaybePromise } from '../interfaces'; +import { BusyStatusTracker } from '../BusyStatusTracker'; /** * Manages all brighterscript projects for the language server */ export class ProjectManager { + constructor() { + this.documentManager.on('flush', (event) => { + void this.applyDocumentChanges(event); + }); + } /** * Collection of all projects @@ -25,17 +31,13 @@ export class ProjectManager { delay: 150 }); - - constructor() { - this.documentManager.on('flush', (event) => { - void this.applyDocumentChanges(event); - }); - } + public busyStatusTracker = new BusyStatusTracker(); /** * Apply all of the queued document changes. This should only be called as a result of the documentManager flushing changes, and never called manually * @param event the document changes that have occurred since the last time we applied */ + @TrackBusyStatus private async applyDocumentChanges(event: FlushEvent) { //apply all of the document actions to each project in parallel await Promise.all(this.projects.map(async (project) => { @@ -50,6 +52,7 @@ export class ProjectManager { * Leave existing projects alone if they are not affected by these changes * @param workspaceConfigs an array of workspaces */ + @TrackBusyStatus public async syncProjects(workspaceConfigs: WorkspaceConfig[]) { //build a list of unique projects across all workspace folders let projectConfigs = (await Promise.all( @@ -132,6 +135,12 @@ export class ProjectManager { return deferred.promise; } + /** + * Get all the semantic tokens for the given file + * @param srcPath absolute path to the file + * @returns an array of semantic tokens + */ + @TrackBusyStatus public async getSemanticTokens(srcPath: string) { //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. const project = await this.findFirstMatchingProject((x) => { @@ -147,6 +156,12 @@ export class ProjectManager { } } + /** + * Get a string containing the transpiled contents of the file at the given path + * @param srcPath path to the file + * @returns the transpiled contents of the file as a string + */ + @TrackBusyStatus public async transpileFile(srcPath: string) { //find the first program that has this file const project = await this.findFirstMatchingProject((p) => { @@ -162,6 +177,12 @@ export class ProjectManager { } } + /** + * Get the completions for the given position in the file + * @param srcPath the path to the file + * @param position the position of the cursor in the file + */ + @TrackBusyStatus public async getCompletions(srcPath: string, position: Position) { // const completions = await Promise.all( // this.projects.map(x => x.getCompletions(srcPath, position)) @@ -177,6 +198,7 @@ export class ProjectManager { * the fastest result will be returned * @returns the hover information or undefined if no hover information was found */ + @TrackBusyStatus public async getHover(srcPath: string, position: Position): Promise { //Ask every project for hover info, keep whichever one responds first that has a valid response let hover = await util.promiseRaceMatch( @@ -193,6 +215,7 @@ export class ProjectManager { * @param position the position of symbol * @returns a list of locations where the symbol under the position is defined in the project */ + @TrackBusyStatus public async getDefinition(srcPath: string, position: Position): Promise { //TODO should we merge definitions across ALL projects? or just return definitions from the first project we found @@ -290,6 +313,7 @@ export class ProjectManager { * Create a project for the given config * @returns a new project, or the existing project if one already exists with this config info */ + @TrackBusyStatus private async createProject(config: ProjectConfig) { //skip this project if we already have it if (this.hasProject(config.projectPath)) { @@ -315,7 +339,6 @@ export class ProjectManager { workspaceFolder: config.workspaceFolder, projectNumber: config.projectNumber ?? ProjectManager.projectNumberSequence++ }); - console.log('Activated'); } public on(eventName: 'critical-failure', handler: (data: { project: LspProject; message: string }) => MaybePromise); @@ -384,3 +407,18 @@ interface ProjectConfig { */ threadingEnabled?: boolean; } + + +/** + * An annotation used to wrap the method in a busyStatus tracking call + */ +function TrackBusyStatus(target: any, propertyKey: string, descriptor: PropertyDescriptor) { + let originalMethod = descriptor.value; + + //wrapping the original method + descriptor.value = function value(this: ProjectManager, ...args: any[]) { + return this.busyStatusTracker.run(() => { + return originalMethod.apply(this, args); + }, originalMethod.name); + }; +} From 4f7964544e0f4d9ec77f9dcd647a42a0a3bffa4c Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 18 Mar 2024 22:49:27 -0400 Subject: [PATCH 030/119] Add signature help --- src/LanguageServer.ts | 39 +++++---------------------- src/Program.ts | 4 +-- src/lsp/LspProject.ts | 8 +++++- src/lsp/Project.ts | 6 +++++ src/lsp/ProjectManager.ts | 25 ++++++++++++++++- src/lsp/worker/WorkerThreadProject.ts | 5 +++- 6 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index c9ceac2f7..2c58ea196 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -179,9 +179,9 @@ export class LanguageServer implements OnHandler { // codeActionProvider: { // codeActionKinds: [CodeActionKind.Refactor] // }, - // signatureHelpProvider: { - // triggerCharacters: ['(', ','] - // }, + signatureHelpProvider: { + triggerCharacters: ['(', ','] + }, definitionProvider: true, hoverProvider: true, executeCommandProvider: { @@ -440,36 +440,9 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage public async onSignatureHelp(params: SignatureHelpParams) { - await this.waitAllProjectFirstRuns(); - - const filepath = util.uriToPath(params.textDocument.uri); - await this.keyedThrottler.onIdleOnce(filepath, true); - - try { - const signatures = util.flatMap( - await Promise.all(this.getProjects().map(project => project.builder.program.getSignatureHelp(filepath, params.position) - )), - c => c - ); - - const activeSignature = signatures.length > 0 ? 0 : null; - - const activeParameter = activeSignature >= 0 ? signatures[activeSignature]?.index : null; - - let results: SignatureHelp = { - signatures: signatures.map((s) => s.signature), - activeSignature: activeSignature, - activeParameter: activeParameter - }; - return results; - } catch (e: any) { - this.connection.console.error(`error in onSignatureHelp: ${e.stack ?? e.message ?? e}`); - return { - signatures: [], - activeSignature: 0, - activeParameter: 0 - }; - } + const filePath = util.uriToPath(params.textDocument.uri); + const result = await this.projectManager.getSignatureHelp(filePath, params.position); + return result; } @AddStackToErrorMessage diff --git a/src/Program.ts b/src/Program.ts index ae308975c..fa282e7e6 100644 --- a/src/Program.ts +++ b/src/Program.ts @@ -1028,8 +1028,8 @@ export class Program { } } - public getSignatureHelp(filepath: string, position: Position): SignatureInfoObj[] { - let file: BrsFile = this.getFile(filepath); + public getSignatureHelp(filePath: string, position: Position): SignatureInfoObj[] { + let file: BrsFile = this.getFile(filePath); if (!file || !isBrsFile(file)) { return []; } diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 2d6cdc969..e498c7ab4 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -2,7 +2,7 @@ import type { Diagnostic, Position, Location } from 'vscode-languageserver'; import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; import type { DocumentAction } from './DocumentManager'; -import type { FileTranspileResult } from '../Program'; +import type { FileTranspileResult, SignatureInfoObj } from '../Program'; /** * Defines the contract between the ProjectManager and the main or worker thread Project classes @@ -79,6 +79,12 @@ export interface LspProject { */ getDefinition(options: { srcPath: string; position: Position }): MaybePromise; + /** + * Get the locations where the symbol at the specified position is defined + * @param options the file path and position to get the definition for + */ + getSignatureHelp(options: { srcPath: string; position: Position }): MaybePromise; + /** * Does this project have the specified file. Should only be called after `.activate()` has completed. */ diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 51ec0a3c8..2287a26e3 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -11,6 +11,7 @@ import { rokuDeploy } from 'roku-deploy'; import type { Location, Position } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; import type { DocumentAction } from './DocumentManager'; +import type { SignatureInfoObj } from '../Program'; export class Project implements LspProject { @@ -258,6 +259,11 @@ export class Project implements LspProject { return this.builder.program.getDefinition(options.srcPath, options.position); } + public async getSignatureHelp(options: { srcPath: string; position: Position }): Promise { + await this.onIdle(); + return this.builder.program.getSignatureHelp(options.srcPath, options.position); + } + /** * Manages the BrighterScript program. The main interface into the compiler/validator */ diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index bd5fcacba..7141e181d 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -5,7 +5,7 @@ import * as EventEmitter from 'eventemitter3'; import type { LspDiagnostic, LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import type { Hover, Position, Location } from 'vscode-languageserver-protocol'; +import type { Hover, Position, Location, SignatureHelp } from 'vscode-languageserver-protocol'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; @@ -228,6 +228,29 @@ export class ProjectManager { return result; } + @TrackBusyStatus + public async getSignatureHelp(srcPath: string, position: Position): Promise { + //Ask every project for definition info, keep whichever one responds first that has a valid response + let signatures = await util.promiseRaceMatch( + this.projects.map(x => x.getSignatureHelp({ srcPath: srcPath, position: position })), + //keep the first non-falsey result + (result) => !!result + ); + + if (signatures?.length > 0) { + const activeSignature = signatures.length > 0 ? 0 : undefined; + + const activeParameter = activeSignature >= 0 ? signatures[activeSignature]?.index : undefined; + + let result: SignatureHelp = { + signatures: signatures.map((s) => s.signature), + activeSignature: activeSignature, + activeParameter: activeParameter + }; + return result; + } + } + /** * Scan a given workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. * If none are found, then the workspaceFolder itself is treated as a project diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index b818859d6..da39df9b3 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -12,7 +12,7 @@ import type { Hover, MaybePromise, SemanticToken } from '../../interfaces'; import type { BsConfig } from '../../BsConfig'; import type { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; -import type { FileTranspileResult } from '../../Program'; +import type { FileTranspileResult, SignatureInfoObj } from '../../Program'; import type { Position, Location } from 'vscode-languageserver-protocol'; export const workerPool = new WorkerPool(() => { @@ -176,6 +176,9 @@ export class WorkerThreadProject implements LspProject { return this.sendStandardRequest('getDefinition', options); } + public async getSignatureHelp(options: { srcPath: string; position: Position }): Promise { + return this.sendStandardRequest('getSignatureHelp', options); + } /** * Handles request/response/update messages from the worker thread From 6ca9d1ead9d08dd4fd50069a58a97ab98ccd48ac Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 19 Mar 2024 14:11:18 -0400 Subject: [PATCH 031/119] Added support for documentSymbol --- src/LanguageServer.ts | 17 +++-------------- src/lsp/DocumentManager.ts | 4 +--- src/lsp/LspProject.ts | 8 +++++++- src/lsp/Project.ts | 10 ++++++---- src/lsp/ProjectManager.ts | 13 ++++++++++++- src/lsp/ReaderWriterManager.ts | 2 +- src/lsp/worker/WorkerThreadProject.ts | 6 +++++- src/util.spec.ts | 10 ++++++++-- src/util.ts | 4 ++-- 9 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 2c58ea196..f4ca8a871 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -14,7 +14,6 @@ import type { SymbolInformation, DocumentSymbolParams, ReferenceParams, - SignatureHelp, SignatureHelpParams, CodeActionParams, SemanticTokens, @@ -45,7 +44,6 @@ import { standardizePath as s, util } from './util'; import { Logger } from './Logger'; import { Throttler } from './Throttler'; import { DiagnosticCollection } from './DiagnosticCollection'; -import { isBrsFile } from './astUtils/reflection'; import { encodeSemanticTokens, semanticTokensLegend } from './SemanticTokenUtils'; import type { WorkspaceConfig } from './lsp/ProjectManager'; import { ProjectManager } from './lsp/ProjectManager'; @@ -169,7 +167,7 @@ export class LanguageServer implements OnHandler { // triggerCharacters: ['.'], // allCommitCharacters: ['.', '@'] // }, - // documentSymbolProvider: true, + documentSymbolProvider: true, // workspaceSymbolProvider: true, semanticTokensProvider: { legend: semanticTokensLegend, @@ -417,17 +415,9 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage public async onDocumentSymbol(params: DocumentSymbolParams) { - await this.waitAllProjectFirstRuns(); - - await this.keyedThrottler.onIdleOnce(util.uriToPath(params.textDocument.uri), true); - const srcPath = util.uriToPath(params.textDocument.uri); - for (const project of this.getProjects()) { - const file = project.builder.program.getFile(srcPath); - if (isBrsFile(file)) { - return file.getDocumentSymbols(); - } - } + const result = await this.projectManager.getDocumentSymbol(srcPath); + return result; } @AddStackToErrorMessage @@ -494,7 +484,6 @@ export class LanguageServer implements OnHandler { /** * Send a new busy status notification to the client based on the current busy status - * @param status */ private sendBusyStatus() { this.busyStatusIndex = ++this.busyStatusIndex <= 0 ? 0 : this.busyStatusIndex; diff --git a/src/lsp/DocumentManager.ts b/src/lsp/DocumentManager.ts index 343499b3d..f88de1cbb 100644 --- a/src/lsp/DocumentManager.ts +++ b/src/lsp/DocumentManager.ts @@ -1,5 +1,5 @@ import * as EventEmitter from 'eventemitter3'; -import type { MaybePromise } from './LspProject'; +import type { MaybePromise } from '../interfaces'; /** * Maintains a queued/buffered list of file operations. These operations don't actually do anything on their own. @@ -25,7 +25,6 @@ export class DocumentManager { /** * Add/set the contents of a file - * @param document */ public set(srcPath: string, fileContents: string) { if (this.queue.has(srcPath)) { @@ -41,7 +40,6 @@ export class DocumentManager { /** * Delete a file - * @param document */ public delete(srcPath: string) { this.queue.delete(srcPath); diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index e498c7ab4..d50312e91 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,4 +1,4 @@ -import type { Diagnostic, Position, Location } from 'vscode-languageserver'; +import type { Diagnostic, Position, Location, DocumentSymbol } from 'vscode-languageserver'; import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; import type { DocumentAction } from './DocumentManager'; @@ -85,6 +85,12 @@ export interface LspProject { */ getSignatureHelp(options: { srcPath: string; position: Position }): MaybePromise; + /** + * Get the list of symbols for the specified file + * @param options + */ + getDocumentSymbol(options: { srcPath: string }): Promise; + /** * Does this project have the specified file. Should only be called after `.activate()` has completed. */ diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 2287a26e3..457c53e1a 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -8,7 +8,7 @@ import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; -import type { Location, Position } from 'vscode-languageserver-protocol'; +import type { DocumentSymbol, Location, Position } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; import type { DocumentAction } from './DocumentManager'; import type { SignatureInfoObj } from '../Program'; @@ -160,8 +160,6 @@ export class Project implements LspProject { /** * Add or replace the in-memory contents of the file at the specified path. This is typically called as the user is typing. * This will cancel any pending validation cycles and queue a future validation cycle instead. - * @param srcPath absolute path to the file - * @param fileContents the contents of the file */ public async applyFileChanges(documentActions: DocumentAction[]): Promise { let didChangeFiles = false; @@ -264,6 +262,11 @@ export class Project implements LspProject { return this.builder.program.getSignatureHelp(options.srcPath, options.position); } + public async getDocumentSymbol(options: { srcPath: string }): Promise { + await this.onIdle(); + return this.builder.program.getDocumentSymbols(options.srcPath); + } + /** * Manages the BrighterScript program. The main interface into the compiler/validator */ @@ -293,7 +296,6 @@ export class Project implements LspProject { /** * Find the path to the bsconfig.json file for this project - * @param config options that help us find the bsconfig.json * @returns path to bsconfig.json, or undefined if unable to find it */ private async getConfigFilePath(config: { configFilePath?: string; projectPath: string }) { diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 7141e181d..8a8024243 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -5,7 +5,7 @@ import * as EventEmitter from 'eventemitter3'; import type { LspDiagnostic, LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import type { Hover, Position, Location, SignatureHelp } from 'vscode-languageserver-protocol'; +import type { Hover, Position, Location, SignatureHelp, DocumentSymbol } from 'vscode-languageserver-protocol'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; @@ -251,6 +251,17 @@ export class ProjectManager { } } + @TrackBusyStatus + public async getDocumentSymbol(srcPath: string): Promise { + //Ask every project for definition info, keep whichever one responds first that has a valid response + let result = await util.promiseRaceMatch( + this.projects.map(x => x.getDocumentSymbol({ srcPath: srcPath })), + //keep the first non-falsey result + (result) => !!result + ); + return result; + } + /** * Scan a given workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. * If none are found, then the workspaceFolder itself is treated as a project diff --git a/src/lsp/ReaderWriterManager.ts b/src/lsp/ReaderWriterManager.ts index 705906173..fe8cf9461 100644 --- a/src/lsp/ReaderWriterManager.ts +++ b/src/lsp/ReaderWriterManager.ts @@ -1,5 +1,5 @@ +import { MaybePromise } from '..'; import { Deferred } from '../deferred'; -import type { MaybePromise } from './LspProject'; /** * Manages multiple readers and writers, and ensures that no readers are reading while a writer is writing. diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index da39df9b3..c290c1777 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -13,7 +13,7 @@ import type { BsConfig } from '../../BsConfig'; import type { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; import type { FileTranspileResult, SignatureInfoObj } from '../../Program'; -import type { Position, Location } from 'vscode-languageserver-protocol'; +import type { Position, Location, DocumentSymbol } from 'vscode-languageserver-protocol'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -180,6 +180,10 @@ export class WorkerThreadProject implements LspProject { return this.sendStandardRequest('getSignatureHelp', options); } + public async getDocumentSymbol(options: { srcPath: string }): Promise { + return this.sendStandardRequest('getDocumentSymbol', options); + } + /** * Handles request/response/update messages from the worker thread */ diff --git a/src/util.spec.ts b/src/util.spec.ts index e1dce2fbf..a11cb64a7 100644 --- a/src/util.spec.ts +++ b/src/util.spec.ts @@ -6,7 +6,7 @@ import type { BsConfig } from './BsConfig'; import * as fsExtra from 'fs-extra'; import { createSandbox } from 'sinon'; import { DiagnosticMessages } from './DiagnosticMessages'; -import { tempDir, rootDir, expectThrows } from './testHelpers.spec'; +import { tempDir, rootDir } from './testHelpers.spec'; import { Program } from './Program'; import type { BsDiagnostic } from '.'; @@ -921,7 +921,7 @@ describe('util', () => { }); }); - describe.only('promiseRaceMatch', () => { + describe('promiseRaceMatch', () => { async function resolveAfter(value: T, timeout: number) { await util.sleep(timeout); return value; @@ -953,6 +953,12 @@ describe('util', () => { ).to.eql('c'); }); + it('does not throw when there were zero promises', async () => { + expect( + await util.promiseRaceMatch([], x => true) + ).to.be.undefined; + }); + it('returns a value even if one of the promises never resolves', async () => { expect( await util.promiseRaceMatch([ diff --git a/src/util.ts b/src/util.ts index dea2847d4..b4c8634e9 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1596,8 +1596,8 @@ export class Util { .map(x => x.error); //if all of them crashed, then reject - if (errors.length === promises.length) { - throw new AggregateError(errors); + if (promises.length > 0 && errors.length === promises.length) { + throw new AggregateError(errors, 'All requests failed. First error message: ' + errors[0].message); } else { //log all of the errors for (const error of errors) { From 533c0cdf267c4045fcc92178d01f6624e2253b04 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 19 Mar 2024 14:39:48 -0400 Subject: [PATCH 032/119] Defer lsp requests until the project manager is ready --- src/lsp/ProjectManager.ts | 116 ++++++++++++++++++++++++++------------ 1 file changed, 81 insertions(+), 35 deletions(-) diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 8a8024243..047b4280a 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -45,6 +45,23 @@ export class ProjectManager { })); } + /** + * A promise that's set when a sync starts, and resolved when the sync is complete + */ + private syncPromise: Promise | undefined; + private firstSync = new Deferred(); + + /** + * Get a promise that resolves when this manager is finished initializing + */ + public onReady() { + return Promise.allSettled([ + //wait for the first sync to finish + this.firstSync.promise, + //make sure we're not in the middle of a sync + this.syncPromise + ]); + } /** * Given a list of all desired projects, create any missing projects and destroy and projects that are no longer available * Treat workspaces that don't have a bsconfig.json as a project. @@ -54,44 +71,52 @@ export class ProjectManager { */ @TrackBusyStatus public async syncProjects(workspaceConfigs: WorkspaceConfig[]) { - //build a list of unique projects across all workspace folders - let projectConfigs = (await Promise.all( - workspaceConfigs.map(async workspaceConfig => { - const projectPaths = await this.getProjectPaths(workspaceConfig); - return projectPaths.map(projectPath => ({ - projectPath: s`${projectPath}`, - workspaceFolder: s`${workspaceConfig}`, - excludePatterns: workspaceConfig.excludePatterns, - threadingEnabled: workspaceConfig.threadingEnabled - })); - }) - )).flat(1); - - //delete projects not represented in the list - for (const project of this.projects) { - //we can't find this existing project in our new list, so scrap it - if (!projectConfigs.find(x => x.projectPath === project.projectPath)) { - this.removeProject(project); + this.syncPromise = (async () => { + //build a list of unique projects across all workspace folders + let projectConfigs = (await Promise.all( + workspaceConfigs.map(async workspaceConfig => { + const projectPaths = await this.getProjectPaths(workspaceConfig); + return projectPaths.map(projectPath => ({ + projectPath: s`${projectPath}`, + workspaceFolder: s`${workspaceConfig}`, + excludePatterns: workspaceConfig.excludePatterns, + threadingEnabled: workspaceConfig.threadingEnabled + })); + }) + )).flat(1); + + //delete projects not represented in the list + for (const project of this.projects) { + //we can't find this existing project in our new list, so scrap it + if (!projectConfigs.find(x => x.projectPath === project.projectPath)) { + this.removeProject(project); + } } - } - // skip projects we already have (they're already loaded...no need to reload them) - projectConfigs = projectConfigs.filter(x => { - return !this.hasProject(x.projectPath); - }); + // skip projects we already have (they're already loaded...no need to reload them) + projectConfigs = projectConfigs.filter(x => { + return !this.hasProject(x.projectPath); + }); + + //dedupe by project path + projectConfigs = [ + ...projectConfigs.reduce( + (acc, x) => acc.set(x.projectPath, x), + new Map() + ).values() + ]; + + //create missing projects + await Promise.all( + projectConfigs.map(config => this.createProject(config)) + ); - //dedupe by project path - projectConfigs = [ - ...projectConfigs.reduce( - (acc, x) => acc.set(x.projectPath, x), - new Map() - ).values() - ]; - - //create missing projects - await Promise.all( - projectConfigs.map(config => this.createProject(config)) - ); + //mark that we've completed our first sync + this.firstSync.tryResolve(); + })(); + + //return the sync promise + return this.syncPromise; } /** @@ -141,6 +166,7 @@ export class ProjectManager { * @returns an array of semantic tokens */ @TrackBusyStatus + @OnReady public async getSemanticTokens(srcPath: string) { //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. const project = await this.findFirstMatchingProject((x) => { @@ -162,6 +188,7 @@ export class ProjectManager { * @returns the transpiled contents of the file as a string */ @TrackBusyStatus + @OnReady public async transpileFile(srcPath: string) { //find the first program that has this file const project = await this.findFirstMatchingProject((p) => { @@ -183,6 +210,7 @@ export class ProjectManager { * @param position the position of the cursor in the file */ @TrackBusyStatus + @OnReady public async getCompletions(srcPath: string, position: Position) { // const completions = await Promise.all( // this.projects.map(x => x.getCompletions(srcPath, position)) @@ -199,6 +227,7 @@ export class ProjectManager { * @returns the hover information or undefined if no hover information was found */ @TrackBusyStatus + @OnReady public async getHover(srcPath: string, position: Position): Promise { //Ask every project for hover info, keep whichever one responds first that has a valid response let hover = await util.promiseRaceMatch( @@ -216,6 +245,7 @@ export class ProjectManager { * @returns a list of locations where the symbol under the position is defined in the project */ @TrackBusyStatus + @OnReady public async getDefinition(srcPath: string, position: Position): Promise { //TODO should we merge definitions across ALL projects? or just return definitions from the first project we found @@ -229,6 +259,7 @@ export class ProjectManager { } @TrackBusyStatus + @OnReady public async getSignatureHelp(srcPath: string, position: Position): Promise { //Ask every project for definition info, keep whichever one responds first that has a valid response let signatures = await util.promiseRaceMatch( @@ -252,7 +283,9 @@ export class ProjectManager { } @TrackBusyStatus + @OnReady public async getDocumentSymbol(srcPath: string): Promise { + await this.onReady(); //Ask every project for definition info, keep whichever one responds first that has a valid response let result = await util.promiseRaceMatch( this.projects.map(x => x.getDocumentSymbol({ srcPath: srcPath })), @@ -456,3 +489,16 @@ function TrackBusyStatus(target: any, propertyKey: string, descriptor: PropertyD }, originalMethod.name); }; } + +/** + * Wraps the method in a an awaited call to `onReady` to ensure the project manager is ready before the method is called + */ +function OnReady(target: any, propertyKey: string, descriptor: PropertyDescriptor) { + let originalMethod = descriptor.value; + + //wrapping the original method + descriptor.value = async function value(this: ProjectManager, ...args: any[]) { + await this.onReady(); + return originalMethod.apply(this, args); + }; +} From f23104806fead6d9a389002dcc7495be1ef919c4 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 19 Mar 2024 15:07:07 -0400 Subject: [PATCH 033/119] Add workspaceSymbol support --- src/LanguageServer.ts | 20 +++-------------- src/lsp/LspProject.ts | 9 +++++++- src/lsp/Project.ts | 7 +++++- src/lsp/ProjectManager.ts | 32 +++++++++++++++++++++++++-- src/lsp/worker/WorkerThreadProject.ts | 6 ++++- 5 files changed, 52 insertions(+), 22 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index f4ca8a871..404ff164e 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -168,7 +168,7 @@ export class LanguageServer implements OnHandler { // allCommitCharacters: ['.', '@'] // }, documentSymbolProvider: true, - // workspaceSymbolProvider: true, + workspaceSymbolProvider: true, semanticTokensProvider: { legend: semanticTokensLegend, full: true @@ -395,22 +395,8 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage public async onWorkspaceSymbol(params: WorkspaceSymbolParams) { - await this.waitAllProjectFirstRuns(); - - const results = util.flatMap( - await Promise.all(this.getProjects().map(project => { - return project.builder.program.getWorkspaceSymbols(); - })), - c => c - ); - - // Remove duplicates - const allSymbols = Object.values(results.reduce((map, symbol) => { - const key = symbol.location.uri + symbol.name; - map[key] = symbol; - return map; - }, {})); - return allSymbols as SymbolInformation[]; + const result = await this.projectManager.getWorkspaceSymbol(); + return result; } @AddStackToErrorMessage diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index d50312e91..07eb7013e 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,4 +1,4 @@ -import type { Diagnostic, Position, Location, DocumentSymbol } from 'vscode-languageserver'; +import type { Diagnostic, Position, Location, DocumentSymbol, SymbolInformation } from 'vscode-languageserver'; import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; import type { DocumentAction } from './DocumentManager'; @@ -91,6 +91,13 @@ export interface LspProject { */ getDocumentSymbol(options: { srcPath: string }): Promise; + /** + * Get the list of symbols for the entire workspace + * @param srcPath + */ + getWorkspaceSymbol(): Promise; + + /** * Does this project have the specified file. Should only be called after `.activate()` has completed. */ diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 457c53e1a..10382d1be 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -8,7 +8,7 @@ import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; -import type { DocumentSymbol, Location, Position } from 'vscode-languageserver-protocol'; +import type { DocumentSymbol, Location, Position, SymbolInformation } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; import type { DocumentAction } from './DocumentManager'; import type { SignatureInfoObj } from '../Program'; @@ -267,6 +267,11 @@ export class Project implements LspProject { return this.builder.program.getDocumentSymbols(options.srcPath); } + public async getWorkspaceSymbol(): Promise { + await this.onIdle(); + return this.builder.program.getWorkspaceSymbols(); + } + /** * Manages the BrighterScript program. The main interface into the compiler/validator */ diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 047b4280a..23f7d544c 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -5,7 +5,7 @@ import * as EventEmitter from 'eventemitter3'; import type { LspDiagnostic, LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import type { Hover, Position, Location, SignatureHelp, DocumentSymbol } from 'vscode-languageserver-protocol'; +import type { Hover, Position, Location, SignatureHelp, DocumentSymbol, SymbolInformation } from 'vscode-languageserver-protocol'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; @@ -285,7 +285,6 @@ export class ProjectManager { @TrackBusyStatus @OnReady public async getDocumentSymbol(srcPath: string): Promise { - await this.onReady(); //Ask every project for definition info, keep whichever one responds first that has a valid response let result = await util.promiseRaceMatch( this.projects.map(x => x.getDocumentSymbol({ srcPath: srcPath })), @@ -295,6 +294,35 @@ export class ProjectManager { return result; } + @TrackBusyStatus + @OnReady + public async getWorkspaceSymbol(): Promise { + //Ask every project for definition info, keep whichever one responds first that has a valid response + let responses = await Promise.allSettled( + this.projects.map(x => x.getWorkspaceSymbol()) + ); + let results = responses + //keep all symbol results + .map((x) => { + return x.status === 'fulfilled' ? x.value : []; + }) + //flatten the array + .flat() + //throw out nulls + .filter(x => !!x); + + // Remove duplicates + const allSymbols = Object.values( + results.reduce((map, symbol) => { + const key = symbol.location.uri + symbol.name; + map[key] = symbol; + return map; + }, {}) + ); + + return allSymbols as SymbolInformation[]; + } + /** * Scan a given workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. * If none are found, then the workspaceFolder itself is treated as a project diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index c290c1777..97d97d45c 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -13,7 +13,7 @@ import type { BsConfig } from '../../BsConfig'; import type { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; import type { FileTranspileResult, SignatureInfoObj } from '../../Program'; -import type { Position, Location, DocumentSymbol } from 'vscode-languageserver-protocol'; +import type { Position, Location, DocumentSymbol, SymbolInformation } from 'vscode-languageserver-protocol'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -184,6 +184,10 @@ export class WorkerThreadProject implements LspProject { return this.sendStandardRequest('getDocumentSymbol', options); } + public async getWorkspaceSymbol(): Promise { + return this.sendStandardRequest('getWorkspaceSymbol'); + } + /** * Handles request/response/update messages from the worker thread */ From cfb8fcbaecd70f32d21daa2c974708957899215f Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 19 Mar 2024 15:20:30 -0400 Subject: [PATCH 034/119] Add onReferences support --- src/LanguageServer.ts | 21 ++++++--------------- src/Program.ts | 4 ++-- src/lsp/LspProject.ts | 8 +++++--- src/lsp/Project.ts | 5 +++++ src/lsp/ProjectManager.ts | 12 ++++++++++++ src/lsp/worker/WorkerThreadProject.ts | 4 ++++ 6 files changed, 34 insertions(+), 20 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 404ff164e..ac862d0d4 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -11,7 +11,6 @@ import type { TextDocumentPositionParams, ExecuteCommandParams, WorkspaceSymbolParams, - SymbolInformation, DocumentSymbolParams, ReferenceParams, SignatureHelpParams, @@ -25,7 +24,8 @@ import type { CompletionParams, ResultProgressReporter, WorkDoneProgressReporter, - SemanticTokensOptions + SemanticTokensOptions, + Location } from 'vscode-languageserver/node'; import { SemanticTokensRequest, @@ -173,7 +173,7 @@ export class LanguageServer implements OnHandler { legend: semanticTokensLegend, full: true } as SemanticTokensOptions, - // referencesProvider: true, + referencesProvider: true, // codeActionProvider: { // codeActionKinds: [CodeActionKind.Refactor] // }, @@ -422,19 +422,10 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - public async onReferences(params: ReferenceParams) { - await this.waitAllProjectFirstRuns(); - - const position = params.position; + public async onReferences(params: ReferenceParams): Promise { const srcPath = util.uriToPath(params.textDocument.uri); - - const results = util.flatMap( - await Promise.all(this.getProjects().map(project => { - return project.builder.program.getReferences(srcPath, position); - })), - c => c - ); - return results.filter((r) => r); + const result = await this.projectManager.getReferences({ srcPath: srcPath, position: params.position }); + return result; } diff --git a/src/Program.ts b/src/Program.ts index 5f1119677..c08eb0484 100644 --- a/src/Program.ts +++ b/src/Program.ts @@ -1,7 +1,7 @@ import * as assert from 'assert'; import * as fsExtra from 'fs-extra'; import * as path from 'path'; -import type { CodeAction, CompletionItem, Position, Range, SignatureInformation, Location, DocumentSymbol, CancellationToken } from 'vscode-languageserver'; +import type { CodeAction, CompletionItem, Position, Range, SignatureInformation, Location, DocumentSymbol, CancellationToken, SymbolInformation } from 'vscode-languageserver'; import { CancellationTokenSource, CompletionItemKind } from 'vscode-languageserver'; import type { BsConfig, FinalizedBsConfig } from './BsConfig'; import { Scope } from './Scope'; @@ -925,7 +925,7 @@ export class Program { /** * Goes through each file and builds a list of workspace symbols for the program. Used by LanguageServer's onWorkspaceSymbol functionality */ - public getWorkspaceSymbols() { + public getWorkspaceSymbols(): SymbolInformation[] { const results = Object.keys(this.files).map(key => { const file = this.files[key]; if (isBrsFile(file)) { diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 07eb7013e..5d84919a6 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -87,16 +87,18 @@ export interface LspProject { /** * Get the list of symbols for the specified file - * @param options */ - getDocumentSymbol(options: { srcPath: string }): Promise; + getDocumentSymbol(options: { srcPath: string }): MaybePromise; /** * Get the list of symbols for the entire workspace - * @param srcPath */ getWorkspaceSymbol(): Promise; + /** + * Get the list of references for the specified file and position + */ + getReferences(options: { srcPath: string; position: Position }): MaybePromise; /** * Does this project have the specified file. Should only be called after `.activate()` has completed. diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 10382d1be..04f0df88b 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -272,6 +272,11 @@ export class Project implements LspProject { return this.builder.program.getWorkspaceSymbols(); } + public async getReferences(options: { srcPath: string; position: Position }): Promise { + await this.onIdle(); + return this.builder.program.getReferences(options.srcPath, options.position); + } + /** * Manages the BrighterScript program. The main interface into the compiler/validator */ diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 23f7d544c..013c2ba19 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -323,6 +323,18 @@ export class ProjectManager { return allSymbols as SymbolInformation[]; } + @TrackBusyStatus + @OnReady + public async getReferences(options: { srcPath: string; position: Position }): Promise { + //Ask every project for definition info, keep whichever one responds first that has a valid response + let result = await util.promiseRaceMatch( + this.projects.map(x => x.getReferences(options)), + //keep the first non-falsey result + (result) => !!result + ); + return result; + } + /** * Scan a given workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. * If none are found, then the workspaceFolder itself is treated as a project diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 97d97d45c..514214611 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -188,6 +188,10 @@ export class WorkerThreadProject implements LspProject { return this.sendStandardRequest('getWorkspaceSymbol'); } + public async getReferences(options: { srcPath: string; position: Position }): Promise { + return this.sendStandardRequest('getReferences', options); + } + /** * Handles request/response/update messages from the worker thread */ From cb507a27b38eb100f7a9771555f1822dc6c46f5a Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 19 Mar 2024 15:26:39 -0400 Subject: [PATCH 035/119] Convert all Project method params to options objects --- src/LanguageServer.ts | 18 +++++++--------- src/lsp/LspProject.ts | 4 ++-- src/lsp/Project.ts | 8 +++---- src/lsp/ProjectManager.ts | 30 +++++++++++++-------------- src/lsp/worker/WorkerThreadProject.ts | 8 +++---- 5 files changed, 33 insertions(+), 35 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index ac862d0d4..bc1b07dd9 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -228,11 +228,9 @@ export class LanguageServer implements OnHandler { */ @AddStackToErrorMessage public async onCompletion1(params: CompletionParams, workDoneProgress: WorkDoneProgressReporter, resultProgress?: ResultProgressReporter) { - const completions = await this.projectManager.getCompletions( - util.uriToPath(params.textDocument.uri), - params.position - ); + const srcPath = util.uriToPath(params.textDocument.uri); + const completions = await this.projectManager.getCompletions({ srcPath: srcPath, position: params.position }); return completions; } @@ -389,7 +387,7 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage public async onHover(params: TextDocumentPositionParams) { const srcPath = util.uriToPath(params.textDocument.uri); - const result = await this.projectManager.getHover(srcPath, params.position); + const result = await this.projectManager.getHover({ srcPath: srcPath, position: params.position }); return result; } @@ -402,7 +400,7 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage public async onDocumentSymbol(params: DocumentSymbolParams) { const srcPath = util.uriToPath(params.textDocument.uri); - const result = await this.projectManager.getDocumentSymbol(srcPath); + const result = await this.projectManager.getDocumentSymbol({ srcPath: srcPath }); return result; } @@ -410,14 +408,14 @@ export class LanguageServer implements OnHandler { public async onDefinition(params: TextDocumentPositionParams) { const srcPath = util.uriToPath(params.textDocument.uri); - const result = this.projectManager.getDefinition(srcPath, params.position); + const result = this.projectManager.getDefinition({ srcPath: srcPath, position: params.position }); return result; } @AddStackToErrorMessage public async onSignatureHelp(params: SignatureHelpParams) { - const filePath = util.uriToPath(params.textDocument.uri); - const result = await this.projectManager.getSignatureHelp(filePath, params.position); + const srcPath = util.uriToPath(params.textDocument.uri); + const result = await this.projectManager.getSignatureHelp({ srcPath: srcPath, position: params.position }); return result; } @@ -432,7 +430,7 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage private async onFullSemanticTokens(params: SemanticTokensParams) { const srcPath = util.uriToPath(params.textDocument.uri); - const result = await this.projectManager.getSemanticTokens(srcPath); + const result = await this.projectManager.getSemanticTokens({ srcPath: srcPath }); return { data: encodeSemanticTokens(result) diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 5d84919a6..5c8b72bcc 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -60,13 +60,13 @@ export interface LspProject { * Get the full list of semantic tokens for the given file path * @param srcPath absolute path to the source file */ - getSemanticTokens(srcPath: string): MaybePromise; + getSemanticTokens(options: { srcPath: string }): MaybePromise; /** * Transpile the specified file * @param srcPath */ - transpileFile(srcPath: string): MaybePromise; + transpileFile(options: { srcPath: string }): MaybePromise; /** * Get the hover information for the specified position in the specified file diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 04f0df88b..54a7fb109 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -237,14 +237,14 @@ export class Project implements LspProject { * Get the full list of semantic tokens for the given file path * @param srcPath absolute path to the source file */ - public async getSemanticTokens(srcPath: string) { + public async getSemanticTokens(options: { srcPath: string }) { await this.onIdle(); - return this.builder.program.getSemanticTokens(srcPath); + return this.builder.program.getSemanticTokens(options.srcPath); } - public async transpileFile(srcPath: string) { + public async transpileFile(options: { srcPath: string }) { await this.onIdle(); - return this.builder.program.getTranspiledFileContents(srcPath); + return this.builder.program.getTranspiledFileContents(options.srcPath); } public async getHover(options: { srcPath: string; position: Position }): Promise { diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 013c2ba19..e8e4e4b1f 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -167,16 +167,16 @@ export class ProjectManager { */ @TrackBusyStatus @OnReady - public async getSemanticTokens(srcPath: string) { + public async getSemanticTokens(options: { srcPath: string }) { //find the first program that has this file, since it would be incredibly inefficient to generate semantic tokens for the same file multiple times. const project = await this.findFirstMatchingProject((x) => { - return x.hasFile(srcPath); + return x.hasFile(options.srcPath); }); //if we found a project if (project) { const result = await Promise.resolve( - project.getSemanticTokens(srcPath) + project.getSemanticTokens(options) ); return result; } @@ -189,16 +189,16 @@ export class ProjectManager { */ @TrackBusyStatus @OnReady - public async transpileFile(srcPath: string) { + public async transpileFile(options: { srcPath: string }) { //find the first program that has this file const project = await this.findFirstMatchingProject((p) => { - return p.hasFile(srcPath); + return p.hasFile(options.srcPath); }); //if we found a project if (project) { const result = await Promise.resolve( - project.transpileFile(srcPath) + project.transpileFile(options) ); return result; } @@ -211,7 +211,7 @@ export class ProjectManager { */ @TrackBusyStatus @OnReady - public async getCompletions(srcPath: string, position: Position) { + public async getCompletions(options: { srcPath: string; position: Position }) { // const completions = await Promise.all( // this.projects.map(x => x.getCompletions(srcPath, position)) // ); @@ -228,10 +228,10 @@ export class ProjectManager { */ @TrackBusyStatus @OnReady - public async getHover(srcPath: string, position: Position): Promise { + public async getHover(options: { srcPath: string; position: Position }): Promise { //Ask every project for hover info, keep whichever one responds first that has a valid response let hover = await util.promiseRaceMatch( - this.projects.map(x => x.getHover({ srcPath: srcPath, position: position })), + this.projects.map(x => x.getHover(options)), //keep the first non-falsey result (result) => !!result ); @@ -246,12 +246,12 @@ export class ProjectManager { */ @TrackBusyStatus @OnReady - public async getDefinition(srcPath: string, position: Position): Promise { + public async getDefinition(options: { srcPath: string; position: Position }): Promise { //TODO should we merge definitions across ALL projects? or just return definitions from the first project we found //Ask every project for definition info, keep whichever one responds first that has a valid response let result = await util.promiseRaceMatch( - this.projects.map(x => x.getDefinition({ srcPath: srcPath, position: position })), + this.projects.map(x => x.getDefinition(options)), //keep the first non-falsey result (result) => !!result ); @@ -260,10 +260,10 @@ export class ProjectManager { @TrackBusyStatus @OnReady - public async getSignatureHelp(srcPath: string, position: Position): Promise { + public async getSignatureHelp(options: { srcPath: string; position: Position }): Promise { //Ask every project for definition info, keep whichever one responds first that has a valid response let signatures = await util.promiseRaceMatch( - this.projects.map(x => x.getSignatureHelp({ srcPath: srcPath, position: position })), + this.projects.map(x => x.getSignatureHelp(options)), //keep the first non-falsey result (result) => !!result ); @@ -284,10 +284,10 @@ export class ProjectManager { @TrackBusyStatus @OnReady - public async getDocumentSymbol(srcPath: string): Promise { + public async getDocumentSymbol(options: { srcPath: string }): Promise { //Ask every project for definition info, keep whichever one responds first that has a valid response let result = await util.promiseRaceMatch( - this.projects.map(x => x.getDocumentSymbol({ srcPath: srcPath })), + this.projects.map(x => x.getDocumentSymbol(options)), //keep the first non-falsey result (result) => !!result ); diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 514214611..afac7e248 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -160,12 +160,12 @@ export class WorkerThreadProject implements LspProject { * Get the full list of semantic tokens for the given file path * @param srcPath absolute path to the source file */ - public async getSemanticTokens(srcPath: string) { - return this.sendStandardRequest('getSemanticTokens', srcPath); + public async getSemanticTokens(options: { srcPath: string }) { + return this.sendStandardRequest('getSemanticTokens', options); } - public async transpileFile(srcPath: string) { - return this.sendStandardRequest('transpileFile', srcPath); + public async transpileFile(options: { srcPath: string }) { + return this.sendStandardRequest('transpileFile', options); } public async getHover(options: { srcPath: string; position: Position }): Promise { From b70a02783e46af64e921e2a871b74b3aa2195bb1 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Fri, 22 Mar 2024 15:01:49 -0400 Subject: [PATCH 036/119] Fix a few issues --- package-lock.json | 4 +-- src/interfaces.ts | 2 +- src/lsp/LspProject.ts | 4 +-- src/lsp/Project.ts | 39 ++++++--------------------- src/lsp/ProjectManager.ts | 4 +-- src/lsp/worker/MessageHandler.ts | 17 +++--------- src/lsp/worker/WorkerThreadProject.ts | 6 ++--- 7 files changed, 22 insertions(+), 54 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7e16df7fc..50fec82c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "brighterscript", - "version": "0.65.26", + "version": "0.65.26-local", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "brighterscript", - "version": "0.65.26", + "version": "0.65.26-local", "license": "MIT", "dependencies": { "@rokucommunity/bslib": "^0.1.1", diff --git a/src/interfaces.ts b/src/interfaces.ts index 49dc3b823..738662f2e 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -1,4 +1,4 @@ -import type { Range, Diagnostic, CodeAction, Position, CompletionItem, Location, DocumentSymbol, WorkspaceSymbol } from 'vscode-languageserver-protocol'; +import type { Range, Diagnostic, CodeAction, Position, CompletionItem, Location, DocumentSymbol, WorkspaceSymbol, Disposable } from 'vscode-languageserver-protocol'; import type { Scope } from './Scope'; import type { BrsFile } from './files/BrsFile'; import type { XmlFile } from './files/XmlFile'; diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 5c8b72bcc..200188cfd 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,4 +1,4 @@ -import type { Diagnostic, Position, Location, DocumentSymbol, SymbolInformation } from 'vscode-languageserver'; +import type { Diagnostic, Position, Location, DocumentSymbol, WorkspaceSymbol } from 'vscode-languageserver'; import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; import type { DocumentAction } from './DocumentManager'; @@ -93,7 +93,7 @@ export interface LspProject { /** * Get the list of symbols for the entire workspace */ - getWorkspaceSymbol(): Promise; + getWorkspaceSymbol(): Promise; /** * Get the list of references for the specified file and position diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 54a7fb109..ca606c667 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -8,7 +8,7 @@ import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; -import type { DocumentSymbol, Location, Position, SymbolInformation } from 'vscode-languageserver-protocol'; +import type { DocumentSymbol, Location, Position, WorkspaceSymbol } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; import type { DocumentAction } from './DocumentManager'; import type { SignatureInfoObj } from '../Program'; @@ -235,7 +235,8 @@ export class Project implements LspProject { /** * Get the full list of semantic tokens for the given file path - * @param srcPath absolute path to the source file + * @param options options for getting semantic tokens + * @param options.srcPath absolute path to the source file */ public async getSemanticTokens(options: { srcPath: string }) { await this.onIdle(); @@ -267,9 +268,12 @@ export class Project implements LspProject { return this.builder.program.getDocumentSymbols(options.srcPath); } - public async getWorkspaceSymbol(): Promise { + public async getWorkspaceSymbol(): Promise { await this.onIdle(); - return this.builder.program.getWorkspaceSymbols(); + console.time('getWorkspaceSymbol'); + const result = this.builder.program.getWorkspaceSymbols(); + console.timeEnd('getWorkspaceSymbol'); + return result; } public async getReferences(options: { srcPath: string; position: Position }): Promise { @@ -374,30 +378,3 @@ export class Project implements LspProject { } } } - -/** - * An annotation used to wrap the method in a readerWriter.write() call - */ -function WriteLock(target: any, propertyKey: string, descriptor: PropertyDescriptor) { - let originalMethod = descriptor.value; - - //wrapping the original method - descriptor.value = function value(this: Project, ...args: any[]) { - return (this as any).readerWriter.write(() => { - return originalMethod.apply(this, args); - }, originalMethod.name); - }; -} -/** - * An annotation used to wrap the method in a readerWriter.read() call - */ -function ReadLock(target: any, propertyKey: string, descriptor: PropertyDescriptor) { - let originalMethod = descriptor.value; - - //wrapping the original method - descriptor.value = function value(this: Project, ...args: any[]) { - return (this as any).readerWriter.read(() => { - return originalMethod.apply(this, args); - }, originalMethod.name); - }; -} diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index e8e4e4b1f..71909dbd1 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -5,7 +5,7 @@ import * as EventEmitter from 'eventemitter3'; import type { LspDiagnostic, LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import type { Hover, Position, Location, SignatureHelp, DocumentSymbol, SymbolInformation } from 'vscode-languageserver-protocol'; +import type { Hover, Position, Location, SignatureHelp, DocumentSymbol, SymbolInformation, WorkspaceSymbol } from 'vscode-languageserver-protocol'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; @@ -296,7 +296,7 @@ export class ProjectManager { @TrackBusyStatus @OnReady - public async getWorkspaceSymbol(): Promise { + public async getWorkspaceSymbol(): Promise { //Ask every project for definition info, keep whichever one responds first that has a valid response let responses = await Promise.allSettled( this.projects.map(x => x.getWorkspaceSymbol()) diff --git a/src/lsp/worker/MessageHandler.ts b/src/lsp/worker/MessageHandler.ts index 118fdeed1..a3fbce894 100644 --- a/src/lsp/worker/MessageHandler.ts +++ b/src/lsp/worker/MessageHandler.ts @@ -75,6 +75,8 @@ export class MessageHandler> { * Send a request to the worker, and wait for a response. * @param name the name of the request * @param options the request options + * @param options.data an array of data that will be passed in as params to the target function + * @param options.id an id for this request */ public async sendRequest(name: TRequestName, options?: { data: any[]; id?: number }) { const request: WorkerMessage = { @@ -117,6 +119,8 @@ export class MessageHandler> { * Send a request to the worker, and wait for a response. * @param name the name of the request * @param options options for the update + * @param options.data an array of data that will be passed in as params to the target function + * @param options.id an id for this update */ public sendUpdate(name: string, options?: { data?: any[]; id?: number }) { let update: WorkerMessage = { @@ -142,19 +146,6 @@ export class MessageHandler> { }; } - /** - * Turn an object with an error structure into a proper error - * @param error the error (in object form) to turn into a proper Error item - */ - private objectToError(error: Error) { - let result = new Error(); - result.name = error.name; - result.message = error.message; - result.stack = error.stack; - result.cause = (error.cause as any)?.message && (error.cause as any)?.stack ? this.objectToError(error.cause as any) : error.cause; - return result; - } - public dispose() { util.applyDispose(this.disposables); } diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index afac7e248..cb2d764f1 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -13,7 +13,7 @@ import type { BsConfig } from '../../BsConfig'; import type { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; import type { FileTranspileResult, SignatureInfoObj } from '../../Program'; -import type { Position, Location, DocumentSymbol, SymbolInformation } from 'vscode-languageserver-protocol'; +import type { Position, Location, DocumentSymbol, WorkspaceSymbol } from 'vscode-languageserver-protocol'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -184,8 +184,8 @@ export class WorkerThreadProject implements LspProject { return this.sendStandardRequest('getDocumentSymbol', options); } - public async getWorkspaceSymbol(): Promise { - return this.sendStandardRequest('getWorkspaceSymbol'); + public async getWorkspaceSymbol(): Promise { + return this.sendStandardRequest('getWorkspaceSymbol'); } public async getReferences(options: { srcPath: string; position: Position }): Promise { From 5eab896f10284f6745d299234afef219fae012ee Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Fri, 22 Mar 2024 15:44:37 -0400 Subject: [PATCH 037/119] Add codeActions support --- src/LanguageServer.ts | 43 ++++++++------------------- src/lsp/LspProject.ts | 7 ++++- src/lsp/Project.ts | 7 ++++- src/lsp/ProjectManager.ts | 21 ++++++++++++- src/lsp/worker/WorkerThreadProject.ts | 6 +++- 5 files changed, 50 insertions(+), 34 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index c3b777d32..e65c392aa 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -34,7 +34,8 @@ import { FileChangeType, ProposedFeatures, TextDocuments, - TextDocumentSyncKind + TextDocumentSyncKind, + CodeActionKind } from 'vscode-languageserver/node'; import { URI } from 'vscode-uri'; import { TextDocument } from 'vscode-languageserver-textdocument'; @@ -174,9 +175,9 @@ export class LanguageServer implements OnHandler { full: true } as SemanticTokensOptions, referencesProvider: true, - // codeActionProvider: { - // codeActionKinds: [CodeActionKind.Refactor] - // }, + codeActionProvider: { + codeActionKinds: [CodeActionKind.Refactor] + }, signatureHelpProvider: { triggerCharacters: ['(', ','] }, @@ -249,32 +250,6 @@ export class LanguageServer implements OnHandler { return item; } - @AddStackToErrorMessage - public async onCodeAction(params: CodeActionParams) { - //ensure programs are initialized - await this.waitAllProjectFirstRuns(); - - let srcPath = util.uriToPath(params.textDocument.uri); - - //wait until the file has settled - await this.onValidateSettled(); - - const codeActions = this - .getProjects() - //skip programs that don't have this file - .filter(x => x.builder?.program?.hasFile(srcPath)) - .flatMap(workspace => workspace.builder.program.getCodeActions(srcPath, params.range)); - - //clone the diagnostics for each code action, since certain diagnostics can have circular reference properties that kill the language server if serialized - for (const codeAction of codeActions) { - if (codeAction.diagnostics) { - codeAction.diagnostics = codeAction.diagnostics?.map(x => util.toDiagnostic(x, params.textDocument.uri)); - } - } - return codeActions; - } - - @AddStackToErrorMessage private async onDidChangeConfiguration() { if (this.hasConfigurationCapability) { @@ -437,6 +412,14 @@ export class LanguageServer implements OnHandler { } as SemanticTokens; } + @AddStackToErrorMessage + public async onCodeAction(params: CodeActionParams) { + const srcPath = util.uriToPath(params.textDocument.uri); + const result = await this.projectManager.getCodeActions({ srcPath: srcPath, range: params.range }); + return result; + } + + @AddStackToErrorMessage public async onExecuteCommand(params: ExecuteCommandParams) { await this.waitAllProjectFirstRuns(); diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 200188cfd..397dd358b 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,4 +1,4 @@ -import type { Diagnostic, Position, Location, DocumentSymbol, WorkspaceSymbol } from 'vscode-languageserver'; +import type { Diagnostic, Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction } from 'vscode-languageserver'; import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; import type { DocumentAction } from './DocumentManager'; @@ -100,6 +100,11 @@ export interface LspProject { */ getReferences(options: { srcPath: string; position: Position }): MaybePromise; + /** + * Get all of the code actions for the specified file and range + */ + getCodeActions(options: { srcPath: string; range: Range }): Promise; + /** * Does this project have the specified file. Should only be called after `.activate()` has completed. */ diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index ca606c667..d8d65d32e 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -8,7 +8,7 @@ import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; -import type { DocumentSymbol, Location, Position, WorkspaceSymbol } from 'vscode-languageserver-protocol'; +import type { CodeAction, DocumentSymbol, Position, Range, Location, WorkspaceSymbol } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; import type { DocumentAction } from './DocumentManager'; import type { SignatureInfoObj } from '../Program'; @@ -281,6 +281,11 @@ export class Project implements LspProject { return this.builder.program.getReferences(options.srcPath, options.position); } + public async getCodeActions(options: { srcPath: string; range: Range }): Promise { + await this.onIdle(); + return this.builder.program.getCodeActions(options.srcPath, options.range); + } + /** * Manages the BrighterScript program. The main interface into the compiler/validator */ diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 71909dbd1..5dd4a967b 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -5,7 +5,7 @@ import * as EventEmitter from 'eventemitter3'; import type { LspDiagnostic, LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import type { Hover, Position, Location, SignatureHelp, DocumentSymbol, SymbolInformation, WorkspaceSymbol } from 'vscode-languageserver-protocol'; +import type { Hover, Position, Range, Location, SignatureHelp, DocumentSymbol, SymbolInformation, WorkspaceSymbol, CodeAction } from 'vscode-languageserver-protocol'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; @@ -335,6 +335,25 @@ export class ProjectManager { return result; } + @TrackBusyStatus + @OnReady + public async getCodeActions(options: { srcPath: string; range: Range }): Promise { + //Ask every project for definition info, keep whichever one responds first that has a valid response + let result = await util.promiseRaceMatch( + this.projects.map(x => x.getCodeActions(options)), + //keep the first non-falsey result + (result) => !!result + ); + + //clone the diagnostics for each code action, since certain diagnostics can have circular reference properties that kill the language server if serialized + for (const codeAction of result) { + if (codeAction.diagnostics) { + codeAction.diagnostics = codeAction.diagnostics?.map(x => util.toDiagnostic(x, options.srcPath)); + } + } + return result; + } + /** * Scan a given workspace for all `bsconfig.json` files. If at least one is found, then only folders who have bsconfig.json are returned. * If none are found, then the workspaceFolder itself is treated as a project diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index cb2d764f1..fe8414ee7 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -13,7 +13,7 @@ import type { BsConfig } from '../../BsConfig'; import type { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; import type { FileTranspileResult, SignatureInfoObj } from '../../Program'; -import type { Position, Location, DocumentSymbol, WorkspaceSymbol } from 'vscode-languageserver-protocol'; +import type { Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction } from 'vscode-languageserver-protocol'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -192,6 +192,10 @@ export class WorkerThreadProject implements LspProject { return this.sendStandardRequest('getReferences', options); } + public async getCodeActions(options: { srcPath: string; range: Range }): Promise { + return this.sendStandardRequest('getCodeActions', options); + } + /** * Handles request/response/update messages from the worker thread */ From 648636f8fd4a16a191d7d9f1f2b0bcdd301bef39 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Fri, 22 Mar 2024 16:09:08 -0400 Subject: [PATCH 038/119] Add completions --- src/LanguageServer.ts | 37 ++++++++------------------- src/lsp/LspProject.ts | 7 ++++- src/lsp/Project.ts | 11 ++++++++ src/lsp/ProjectManager.ts | 18 ++++++------- src/lsp/worker/WorkerThreadProject.ts | 7 +++-- 5 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index e65c392aa..2c1f5cf97 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -25,7 +25,9 @@ import type { ResultProgressReporter, WorkDoneProgressReporter, SemanticTokensOptions, - Location + Location, + CompletionList, + CancellationToken } from 'vscode-languageserver/node'; import { SemanticTokensRequest, @@ -161,13 +163,13 @@ export class LanguageServer implements OnHandler { return { capabilities: { textDocumentSync: TextDocumentSyncKind.Full, - // // Tell the client that the server supports code completion - // completionProvider: { - // resolveProvider: true, - // //anytime the user types a period, auto-show the completion results - // triggerCharacters: ['.'], - // allCommitCharacters: ['.', '@'] - // }, + // Tell the client that the server supports code completion + completionProvider: { + resolveProvider: false, + //anytime the user types a period, auto-show the completion results + triggerCharacters: ['.'], + allCommitCharacters: ['.', '@'] + }, documentSymbolProvider: true, workspaceSymbolProvider: true, semanticTokensProvider: { @@ -228,28 +230,12 @@ export class LanguageServer implements OnHandler { * Provide a list of completion items based on the current cursor position */ @AddStackToErrorMessage - public async onCompletion1(params: CompletionParams, workDoneProgress: WorkDoneProgressReporter, resultProgress?: ResultProgressReporter) { + public async onCompletion(params: CompletionParams, token: CancellationToken, workDoneProgress: WorkDoneProgressReporter, resultProgress: ResultProgressReporter): Promise { const srcPath = util.uriToPath(params.textDocument.uri); - const completions = await this.projectManager.getCompletions({ srcPath: srcPath, position: params.position }); return completions; } - /** - * Provide a full completion item from the selection - */ - @AddStackToErrorMessage - public onCompletionResolve(item: CompletionItem): CompletionItem { - if (item.data === 1) { - item.detail = 'TypeScript details'; - item.documentation = 'TypeScript documentation'; - } else if (item.data === 2) { - item.detail = 'JavaScript details'; - item.documentation = 'JavaScript documentation'; - } - return item; - } - @AddStackToErrorMessage private async onDidChangeConfiguration() { if (this.hasConfigurationCapability) { @@ -810,7 +796,6 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage private onTextDocumentDidChangeContent(event: TextDocumentChangeEvent) { const srcPath = URI.parse(event.document.uri).fsPath; - console.log('setFile', srcPath); this.projectManager.setFile(srcPath, event.document.getText()); } diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 397dd358b..4320e4457 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,4 +1,4 @@ -import type { Diagnostic, Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction } from 'vscode-languageserver'; +import type { Diagnostic, Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction, CompletionList } from 'vscode-languageserver'; import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; import type { BsConfig } from '../BsConfig'; import type { DocumentAction } from './DocumentManager'; @@ -105,6 +105,11 @@ export interface LspProject { */ getCodeActions(options: { srcPath: string; range: Range }): Promise; + /** + * Get the completions for the specified file and position + */ + getCompletions(options: { srcPath: string; position: Position }): Promise; + /** * Does this project have the specified file. Should only be called after `.activate()` has completed. */ diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index d8d65d32e..e7bfe889c 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -9,6 +9,7 @@ import { URI } from 'vscode-uri'; import { Deferred } from '../deferred'; import { rokuDeploy } from 'roku-deploy'; import type { CodeAction, DocumentSymbol, Position, Range, Location, WorkspaceSymbol } from 'vscode-languageserver-protocol'; +import { CompletionList } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; import type { DocumentAction } from './DocumentManager'; import type { SignatureInfoObj } from '../Program'; @@ -286,6 +287,16 @@ export class Project implements LspProject { return this.builder.program.getCodeActions(options.srcPath, options.range); } + public async getCompletions(options: { srcPath: string; position: Position }): Promise { + await this.onIdle(); + const completions = this.builder.program.getCompletions(options.srcPath, options.position); + const result = CompletionList.create(completions); + result.itemDefaults = { + commitCharacters: ['.'] + }; + return result; + } + /** * Manages the BrighterScript program. The main interface into the compiler/validator */ diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 5dd4a967b..9dbb0f9e7 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -5,7 +5,7 @@ import * as EventEmitter from 'eventemitter3'; import type { LspDiagnostic, LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import type { Hover, Position, Range, Location, SignatureHelp, DocumentSymbol, SymbolInformation, WorkspaceSymbol, CodeAction } from 'vscode-languageserver-protocol'; +import type { Hover, Position, Range, Location, SignatureHelp, DocumentSymbol, SymbolInformation, WorkspaceSymbol, CodeAction, CompletionList } from 'vscode-languageserver-protocol'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; @@ -211,14 +211,14 @@ export class ProjectManager { */ @TrackBusyStatus @OnReady - public async getCompletions(options: { srcPath: string; position: Position }) { - // const completions = await Promise.all( - // this.projects.map(x => x.getCompletions(srcPath, position)) - // ); - - // for (let completion of completions) { - // completion.commitCharacters = ['.']; - // } + public async getCompletions(options: { srcPath: string; position: Position }): Promise { + //Ask every project for results, keep whichever one responds first that has a valid response + let result = await util.promiseRaceMatch( + this.projects.map(x => x.getCompletions(options)), + //keep the first non-falsey result + (result) => !!result + ); + return result; } /** diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index fe8414ee7..26c3dee25 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -13,7 +13,7 @@ import type { BsConfig } from '../../BsConfig'; import type { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; import type { FileTranspileResult, SignatureInfoObj } from '../../Program'; -import type { Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction } from 'vscode-languageserver-protocol'; +import type { Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction, CompletionList } from 'vscode-languageserver-protocol'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -158,7 +158,6 @@ export class WorkerThreadProject implements LspProject { /** * Get the full list of semantic tokens for the given file path - * @param srcPath absolute path to the source file */ public async getSemanticTokens(options: { srcPath: string }) { return this.sendStandardRequest('getSemanticTokens', options); @@ -196,6 +195,10 @@ export class WorkerThreadProject implements LspProject { return this.sendStandardRequest('getCodeActions', options); } + public async getCompletions(options: { srcPath: string; position: Position }): Promise { + return this.sendStandardRequest('getCompletions', options); + } + /** * Handles request/response/update messages from the worker thread */ From 9892c69ecb1b96266d003d90c3aca99150447391 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Fri, 22 Mar 2024 17:13:34 -0400 Subject: [PATCH 039/119] Reload server when the user changes workspace settings --- src/LanguageServer.ts | 33 ++++++++++++++++++--------------- src/lsp/ProjectManager.ts | 11 +++++++++-- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 2c1f5cf97..4302783d0 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -27,7 +27,9 @@ import type { SemanticTokensOptions, Location, CompletionList, - CancellationToken + CancellationToken, + DidChangeConfigurationParams, + DidChangeConfigurationRegistrationOptions } from 'vscode-languageserver/node'; import { SemanticTokensRequest, @@ -201,16 +203,23 @@ export class LanguageServer implements OnHandler { public async onInitialized() { try { if (this.hasConfigurationCapability) { - // Register for all configuration changes. + // register for when the user changes workspace or user settings await this.connection.client.register( DidChangeConfigurationNotification.type, - undefined + { + //we only care about when these settings sections change + section: [ + 'brightscript', + 'files' + ] + } as DidChangeConfigurationRegistrationOptions ); } await this.syncProjects(); if (this.clientHasWorkspaceFolderCapability) { + //if the client changes their workspaces, we need to get our projects in sync this.connection.workspace.onDidChangeWorkspaceFolders(async (evt) => { await this.syncProjects(); }); @@ -237,16 +246,9 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - private async onDidChangeConfiguration() { - if (this.hasConfigurationCapability) { - //if the user changes any config value, just mass-reload all projects - await this.reloadProjects(this.getProjects()); - // Reset all cached document settings - } else { - // this.globalSettings = ( - // (change.settings.languageServerExample || this.defaultSettings) - // ); - } + protected async onDidChangeConfiguration(args: DidChangeConfigurationParams) { + //if the user changes any user/workspace config settings, just mass-reload all projects + await this.syncProjects(true); } /** @@ -469,8 +471,9 @@ export class LanguageServer implements OnHandler { * Treat workspaces that don't have a bsconfig.json as a project. * Handle situations where bsconfig.json files were added or removed (to elevate/lower workspaceFolder projects accordingly) * Leave existing projects alone if they are not affected by these changes + * @param forceReload if true, all projects are discarded and recreated from scratch */ - private async syncProjects() { + private async syncProjects(forceReload = false) { // get all workspace paths from the client let workspaces = await Promise.all( (await this.connection.workspace.getWorkspaceFolders() ?? []).map(async (x) => { @@ -487,7 +490,7 @@ export class LanguageServer implements OnHandler { }) ); - await this.projectManager.syncProjects(workspaces); + await this.projectManager.syncProjects(workspaces, forceReload); } /** diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 9dbb0f9e7..133bafb73 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -70,7 +70,14 @@ export class ProjectManager { * @param workspaceConfigs an array of workspaces */ @TrackBusyStatus - public async syncProjects(workspaceConfigs: WorkspaceConfig[]) { + public async syncProjects(workspaceConfigs: WorkspaceConfig[], forceReload = false) { + //if we're force reloading, destroy all projects and start fresh + if (forceReload) { + for (const project of this.projects) { + this.removeProject(project); + } + } + this.syncPromise = (async () => { //build a list of unique projects across all workspace folders let projectConfigs = (await Promise.all( @@ -78,7 +85,7 @@ export class ProjectManager { const projectPaths = await this.getProjectPaths(workspaceConfig); return projectPaths.map(projectPath => ({ projectPath: s`${projectPath}`, - workspaceFolder: s`${workspaceConfig}`, + workspaceFolder: s`${workspaceConfig.workspaceFolder}`, excludePatterns: workspaceConfig.excludePatterns, threadingEnabled: workspaceConfig.threadingEnabled })); From 812b989d1f5f2539fac41c11352b8a4c2d3ac517 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sun, 24 Mar 2024 07:18:18 -0400 Subject: [PATCH 040/119] Fix bug where only the first file change would be applied in each batch --- src/lsp/DocumentManager.ts | 3 +++ src/lsp/Project.ts | 15 ++++++------ src/lsp/ProjectManager.spec.ts | 45 +++++++++++++++++++++++++++++++++- src/lsp/ProjectManager.ts | 7 ++++++ 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/src/lsp/DocumentManager.ts b/src/lsp/DocumentManager.ts index f88de1cbb..d3e8a6d8a 100644 --- a/src/lsp/DocumentManager.ts +++ b/src/lsp/DocumentManager.ts @@ -1,5 +1,6 @@ import * as EventEmitter from 'eventemitter3'; import type { MaybePromise } from '../interfaces'; +import util from '../util'; /** * Maintains a queued/buffered list of file operations. These operations don't actually do anything on their own. @@ -27,6 +28,7 @@ export class DocumentManager { * Add/set the contents of a file */ public set(srcPath: string, fileContents: string) { + srcPath = util.standardizePath(srcPath); if (this.queue.has(srcPath)) { this.queue.delete(srcPath); } @@ -42,6 +44,7 @@ export class DocumentManager { * Delete a file */ public delete(srcPath: string) { + srcPath = util.standardizePath(srcPath); this.queue.delete(srcPath); this.queue.set(srcPath, { type: 'delete', srcPath: srcPath }); } diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index e7bfe889c..77aabe407 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -45,8 +45,9 @@ export class Project implements LspProject { this.builder.plugins.add({ name: 'bsc-language-server', afterProgramValidate: () => { + const diagnostics = this.getDiagnostics(); this.emit('diagnostics', { - diagnostics: this.getDiagnostics() + diagnostics: diagnostics }); } } as CompilerPlugin); @@ -77,9 +78,6 @@ export class Project implements LspProject { //trigger a validation (but don't wait for it. That way we can cancel it sooner if we get new incoming data or requests) void this.validate(); - //flush any diagnostics generated by this initial run - this.emit('diagnostics', { diagnostics: this.getDiagnostics() }); - this.activationDeferred.resolve(); } @@ -131,7 +129,8 @@ export class Project implements LspProject { private activationDeferred: Deferred; public getDiagnostics() { - return this.builder.getDiagnostics().map(x => { + const diagnostics = this.builder.getDiagnostics(); + return diagnostics.map(x => { const uri = URI.file(x.file.srcPath).toString(); return { ...util.toDiagnostic(x, uri), @@ -165,13 +164,15 @@ export class Project implements LspProject { public async applyFileChanges(documentActions: DocumentAction[]): Promise { let didChangeFiles = false; for (const action of documentActions) { + let didChangeThisFile = false; if (this.hasFile(action.srcPath)) { if (action.type === 'set') { - didChangeFiles ||= this.setFile(action.srcPath, action.fileContents); + didChangeThisFile = this.setFile(action.srcPath, action.fileContents); } else if (action.type === 'delete') { - didChangeFiles ||= this.removeFile(action.srcPath); + didChangeThisFile = this.removeFile(action.srcPath); } } + didChangeFiles = didChangeFiles || didChangeThisFile; } if (didChangeFiles) { await this.validate(); diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index cdf576286..1286d660b 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -1,12 +1,14 @@ import { expect } from 'chai'; import { ProjectManager } from './ProjectManager'; -import { tempDir, rootDir } from '../testHelpers.spec'; +import { tempDir, rootDir, expectZeroDiagnostics, expectDiagnostics } from '../testHelpers.spec'; import * as fsExtra from 'fs-extra'; import util, { standardizePath as s } from '../util'; import { createSandbox } from 'sinon'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; import { getWakeWorkerThreadPromise } from './worker/WorkerThreadProject.spec'; +import type { LspDiagnostic } from './LspProject'; +import { DiagnosticMessages } from '../DiagnosticMessages'; const sinon = createSandbox(); describe('ProjectManager', () => { @@ -16,13 +18,31 @@ describe('ProjectManager', () => { manager = new ProjectManager(); fsExtra.emptyDirSync(tempDir); sinon.restore(); + diagnosticsListeners = []; + diagnostics = []; + manager.on('diagnostics', (event) => { + diagnostics.push(event.diagnostics); + diagnosticsListeners.pop()?.(event.diagnostics); + }); }); afterEach(() => { fsExtra.emptyDirSync(tempDir); sinon.restore(); + manager.dispose(); }); + /** + * Get a promise that resolves when the next diagnostics event is emitted + */ + function onNextDiagnostics() { + return new Promise((resolve) => { + diagnosticsListeners.push(resolve); + }); + } + let diagnosticsListeners: Array<(diagnostics: LspDiagnostic[]) => void> = []; + let diagnostics: Array = []; + describe('on', () => { it('emits events', async () => { @@ -156,6 +176,29 @@ describe('ProjectManager', () => { }); }); + describe('onDidChangeWatchedFiles', () => { + it('properly syncs changes', async () => { + fsExtra.outputFileSync(`${rootDir}/source/lib1.brs`, `sub test1():print "alpha":end sub`); + fsExtra.outputFileSync(`${rootDir}/source/lib2.brs`, `sub test2():print "beta":end sub`); + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + expectZeroDiagnostics(await onNextDiagnostics()); + + manager.setFile(`${rootDir}/source/lib1.brs`, `sub test1():print alpha:end sub`); + manager.setFile(`${rootDir}/source/lib2.brs`, `sub test2()::print beta:end sub`); + + expectDiagnostics(await onNextDiagnostics(), [ + DiagnosticMessages.cannotFindName('alpha').message, + DiagnosticMessages.cannotFindName('beta').message + ]); + + manager.setFile(`${rootDir}/source/lib1.brs`, `sub test1():print "alpha":end sub`); + manager.setFile(`${rootDir}/source/lib2.brs`, `sub test2():print "beta":end sub`); + expectZeroDiagnostics(await onNextDiagnostics()); + }); + }); + describe('threading', () => { before(async function workerThreadWarmup() { this.timeout(20_000); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 133bafb73..0b55a1851 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -491,6 +491,13 @@ export class ProjectManager { this.emitter.emit(eventName, data); } private emitter = new EventEmitter(); + + public dispose() { + this.emitter.removeAllListeners(); + for (const project of this.projects) { + project.dispose(); + } + } } export interface WorkspaceConfig { From 4d64647862434613af8c4165fe9ce0502336c784 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sun, 24 Mar 2024 14:03:41 -0400 Subject: [PATCH 041/119] Implement onDidChangeWatchedFiles --- src/LanguageServer.ts | 31 ++++++++------- src/ProgramBuilder.ts | 4 +- src/interfaces.ts | 11 +++++- src/lsp/DocumentManager.ts | 11 +++++- src/lsp/Project.ts | 9 ++++- src/lsp/ProjectManager.spec.ts | 41 ++++++++++++++++--- src/lsp/ProjectManager.ts | 72 +++++++++++++++++++++++++++++----- 7 files changed, 146 insertions(+), 33 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 4302783d0..fa02e7a7e 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -39,7 +39,9 @@ import { ProposedFeatures, TextDocuments, TextDocumentSyncKind, - CodeActionKind + CodeActionKind, + DidChangeWatchedFilesNotification, + WatchKind } from 'vscode-languageserver/node'; import { URI } from 'vscode-uri'; import { TextDocument } from 'vscode-languageserver-textdocument'; @@ -258,9 +260,14 @@ export class LanguageServer implements OnHandler { * file types are watched (.brs,.bs,.xml,manifest, and any json/text/image files) */ @AddStackToErrorMessage - private async onDidChangeWatchedFiles(params: DidChangeWatchedFilesParams) { - //ensure programs are initialized - await this.waitAllProjectFirstRuns(); + protected async onDidChangeWatchedFiles(params: DidChangeWatchedFilesParams) { + await this.projectManager.handleFileChanges( + params.changes.map(x => ({ + srcPath: util.uriToPath(x.uri), + type: x.type + })) + ); + return; let projects = this.getProjects(); @@ -524,13 +531,6 @@ export class LanguageServer implements OnHandler { void this.connection.sendNotification('critical-failure', message); } - /** - * Wait for all programs' first run to complete - */ - private async waitAllProjectFirstRuns(waitForFirstProject = true) { - //TODO delete me - } - /** * Event handler for when the program wants to load file contents. * anytime the program wants to load a file, check with our in-memory document cache first @@ -797,9 +797,12 @@ export class LanguageServer implements OnHandler { } @AddStackToErrorMessage - private onTextDocumentDidChangeContent(event: TextDocumentChangeEvent) { - const srcPath = URI.parse(event.document.uri).fsPath; - this.projectManager.setFile(srcPath, event.document.getText()); + private async onTextDocumentDidChangeContent(event: TextDocumentChangeEvent) { + await this.projectManager.handleFileChanges([{ + srcPath: URI.parse(event.document.uri).fsPath, + type: FileChangeType.Changed, + fileContents: event.document.getText() + }]); } private async validateAll() { diff --git a/src/ProgramBuilder.ts b/src/ProgramBuilder.ts index 8f63df296..e11cac07f 100644 --- a/src/ProgramBuilder.ts +++ b/src/ProgramBuilder.ts @@ -269,7 +269,7 @@ export class ProgramBuilder { /** * Run the entire process exactly one time. */ - private runOnce(options?: { skipValidation?: boolean; }) { + private runOnce(options?: { skipValidation?: boolean }) { //clear the console this.clearConsole(); let cancellationToken = { isCanceled: false }; @@ -355,7 +355,7 @@ export class ProgramBuilder { * Run the process once, allowing cancelability. * NOTE: This should only be called by `runOnce`. */ - private async _runOnce(options: { cancellationToken: { isCanceled: any }; skipValidation: boolean; }) { + private async _runOnce(options: { cancellationToken: { isCanceled: any }; skipValidation: boolean }) { let wereDiagnosticsPrinted = false; try { //maybe cancel? diff --git a/src/interfaces.ts b/src/interfaces.ts index 738662f2e..17bfc5f78 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -1,4 +1,4 @@ -import type { Range, Diagnostic, CodeAction, Position, CompletionItem, Location, DocumentSymbol, WorkspaceSymbol, Disposable } from 'vscode-languageserver-protocol'; +import type { Range, Diagnostic, CodeAction, Position, CompletionItem, Location, DocumentSymbol, WorkspaceSymbol, Disposable, FileChangeType } from 'vscode-languageserver-protocol'; import type { Scope } from './Scope'; import type { BrsFile } from './files/BrsFile'; import type { XmlFile } from './files/XmlFile'; @@ -550,3 +550,12 @@ export interface FileLink { export type DisposableLike = Disposable | (() => any); export type MaybePromise = T | Promise; + +export interface FileChange { + srcPath: string; + type: FileChangeType; + /** + * If provided, this is the new contents of the file. If not provided, the file will be read from disk + */ + fileContents?: string; +} diff --git a/src/lsp/DocumentManager.ts b/src/lsp/DocumentManager.ts index d3e8a6d8a..7354f9a30 100644 --- a/src/lsp/DocumentManager.ts +++ b/src/lsp/DocumentManager.ts @@ -41,11 +41,20 @@ export class DocumentManager { } /** - * Delete a file + * Delete a file or directory. If a directory is provided, all pending changes within that directory will be discarded + * and only the delete action will be queued */ public delete(srcPath: string) { srcPath = util.standardizePath(srcPath); + //remove any pending action with this exact path this.queue.delete(srcPath); + //we can't tell if this a directory, so just remove all pending changes for files that start with this path + for (const key of this.queue.keys()) { + if (key.startsWith(srcPath)) { + this.queue.delete(key); + } + } + //register this delete this.queue.set(srcPath, { type: 'delete', srcPath: srcPath }); } diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 77aabe407..5eb84d0ae 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -165,7 +165,7 @@ export class Project implements LspProject { let didChangeFiles = false; for (const action of documentActions) { let didChangeThisFile = false; - if (this.hasFile(action.srcPath)) { + if (this.willAcceptFile(action.srcPath)) { if (action.type === 'set') { didChangeThisFile = this.setFile(action.srcPath, action.fileContents); } else if (action.type === 'delete') { @@ -180,6 +180,13 @@ export class Project implements LspProject { return didChangeFiles; } + /** + * Determine if this project will accept the file at the specified path (i.e. does it match a pattern in the project's files array) + */ + private willAcceptFile(srcPath: string) { + return !!rokuDeploy.getDestPath(srcPath, this.builder.program.options.files, this.builder.program.options.rootDir); + } + /** * Set new contents for a file. This is safe to call any time. Changes will be queued and flushed at the correct times * during the program's lifecycle flow diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index 1286d660b..be08f5d5f 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -9,6 +9,7 @@ import { WorkerThreadProject } from './worker/WorkerThreadProject'; import { getWakeWorkerThreadPromise } from './worker/WorkerThreadProject.spec'; import type { LspDiagnostic } from './LspProject'; import { DiagnosticMessages } from '../DiagnosticMessages'; +import { FileChangeType } from 'vscode-languageserver-protocol'; const sinon = createSandbox(); describe('ProjectManager', () => { @@ -176,7 +177,7 @@ describe('ProjectManager', () => { }); }); - describe('onDidChangeWatchedFiles', () => { + describe('handleFileChanges', () => { it('properly syncs changes', async () => { fsExtra.outputFileSync(`${rootDir}/source/lib1.brs`, `sub test1():print "alpha":end sub`); fsExtra.outputFileSync(`${rootDir}/source/lib2.brs`, `sub test2():print "beta":end sub`); @@ -185,17 +186,47 @@ describe('ProjectManager', () => { }]); expectZeroDiagnostics(await onNextDiagnostics()); - manager.setFile(`${rootDir}/source/lib1.brs`, `sub test1():print alpha:end sub`); - manager.setFile(`${rootDir}/source/lib2.brs`, `sub test2()::print beta:end sub`); + await manager.handleFileChanges([ + { srcPath: `${rootDir}/source/lib1.brs`, fileContents: `sub test1():print alpha:end sub`, type: FileChangeType.Changed }, + { srcPath: `${rootDir}/source/lib2.brs`, fileContents: `sub test2()::print beta:end sub`, type: FileChangeType.Changed } + ]); expectDiagnostics(await onNextDiagnostics(), [ DiagnosticMessages.cannotFindName('alpha').message, DiagnosticMessages.cannotFindName('beta').message ]); - manager.setFile(`${rootDir}/source/lib1.brs`, `sub test1():print "alpha":end sub`); - manager.setFile(`${rootDir}/source/lib2.brs`, `sub test2():print "beta":end sub`); + await manager.handleFileChanges([ + { srcPath: `${rootDir}/source/lib1.brs`, fileContents: `sub test1():print "alpha":end sub`, type: FileChangeType.Changed }, + { srcPath: `${rootDir}/source/lib2.brs`, fileContents: `sub test2()::print "beta":end sub`, type: FileChangeType.Changed } + ]); + + expectZeroDiagnostics(await onNextDiagnostics()); + }); + + it('adds all new files in a folder', async () => { + fsExtra.outputFileSync(`${rootDir}/source/main.brs`, `sub main():print "main":end sub`); + + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); expectZeroDiagnostics(await onNextDiagnostics()); + + //add a few files to a folder, then register that folder as an "add" + fsExtra.outputFileSync(`${rootDir}/source/libs/alpha/beta.brs`, `sub beta(): print one: end sub`); + fsExtra.outputFileSync(`${rootDir}/source/libs/alpha/charlie/delta.brs`, `sub delta():print two:end sub`); + fsExtra.outputFileSync(`${rootDir}/source/libs/echo/foxtrot.brs`, `sub foxtrot():print three:end sub`); + + await manager.handleFileChanges([ + //register the entire folder as an "add" + { srcPath: `${rootDir}/source/libs`, type: FileChangeType.Created } + ]); + + expectDiagnostics(await onNextDiagnostics(), [ + DiagnosticMessages.cannotFindName('one').message, + DiagnosticMessages.cannotFindName('two').message, + DiagnosticMessages.cannotFindName('three').message + ]); }); }); diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index 0b55a1851..ca0cc3fb9 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -1,16 +1,18 @@ import { standardizePath as s, util } from '../util'; import { rokuDeploy } from 'roku-deploy'; +import * as fsExtra from 'fs-extra'; import * as path from 'path'; import * as EventEmitter from 'eventemitter3'; import type { LspDiagnostic, LspProject } from './LspProject'; import { Project } from './Project'; import { WorkerThreadProject } from './worker/WorkerThreadProject'; -import type { Hover, Position, Range, Location, SignatureHelp, DocumentSymbol, SymbolInformation, WorkspaceSymbol, CodeAction, CompletionList } from 'vscode-languageserver-protocol'; +import { type Hover, type Position, type Range, type Location, type SignatureHelp, type DocumentSymbol, type SymbolInformation, type WorkspaceSymbol, type CodeAction, type CompletionList, FileChangeType } from 'vscode-languageserver-protocol'; import { Deferred } from '../deferred'; import type { FlushEvent } from './DocumentManager'; import { DocumentManager } from './DocumentManager'; -import type { MaybePromise } from '../interfaces'; +import type { FileChange, MaybePromise } from '../interfaces'; import { BusyStatusTracker } from '../BusyStatusTracker'; +import * as fastGlob from 'fast-glob'; /** * Manages all brighterscript projects for the language server @@ -132,10 +134,68 @@ export class ProjectManager { * @param srcPath absolute source path of the file * @param fileContents the text contents of the file */ - public setFile(srcPath: string, fileContents: string) { + private setFile(srcPath: string, fileContents: string) { this.documentManager.set(srcPath, fileContents); } + /** + * Promise that resolves when all file changes have been processed (so we can queue file changes in sequence) + */ + private handleFileChangesPromise: Promise = Promise.resolve(); + + /** + * Handle when files or directories are added, changed, or deleted in the workspace. + * This is safe to call any time. Changes will be queued and flushed at the correct times + */ + public async handleFileChanges(changes: FileChange[]) { + //wait for the previous file change handling to finish, then handle these changes + this.handleFileChangesPromise = this.handleFileChangesPromise.catch((e) => { + console.error(e); + //ignore errors, they will be handled by the previous caller + }).then(() => { + //process all file changes in parallel + return Promise.all(changes.map(async (change) => { + await this.handleFileChange(change); + })); + }); + await this.handleFileChangesPromise; + return this.handleFileChangesPromise; + } + + /** + * Handle a single file change. If the file is a directory, this will recursively read all files in the directory and call `handleFileChanges` again + */ + private async handleFileChange(change: FileChange) { + const srcPath = util.standardizePath(change.srcPath); + if (change.type === FileChangeType.Deleted) { + //mark this document or directory as deleted + this.documentManager.delete(srcPath); + + //file added or changed + } else { + //this is a new file. set the file contents + if (fsExtra.statSync(srcPath).isFile()) { + const fileContents = change.fileContents ?? (await fsExtra.readFile(change.srcPath, 'utf8')).toString(); + this.setFile(change.srcPath, fileContents); + + //if this is a new directory, read all files recursively and register those as file changes too + } else { + const files = await fastGlob('**/*', { + cwd: change.srcPath, + onlyFiles: true, + absolute: true + }); + //pipe all files found recursively in the new directory through this same function so they can be processed correctly + await Promise.all(files.map((srcPath) => { + return this.handleFileChange({ + srcPath: srcPath, + type: FileChangeType.Changed + }); + })); + } + } + } + /** * Return the first project where the async matcher returns true */ @@ -169,7 +229,6 @@ export class ProjectManager { /** * Get all the semantic tokens for the given file - * @param srcPath absolute path to the file * @returns an array of semantic tokens */ @TrackBusyStatus @@ -191,7 +250,6 @@ export class ProjectManager { /** * Get a string containing the transpiled contents of the file at the given path - * @param srcPath path to the file * @returns the transpiled contents of the file as a string */ @TrackBusyStatus @@ -213,8 +271,6 @@ export class ProjectManager { /** * Get the completions for the given position in the file - * @param srcPath the path to the file - * @param position the position of the cursor in the file */ @TrackBusyStatus @OnReady @@ -247,8 +303,6 @@ export class ProjectManager { /** * Get the definition for the symbol at the given position in the file - * @param srcPath the path to the file - * @param position the position of symbol * @returns a list of locations where the symbol under the position is defined in the project */ @TrackBusyStatus From d47834b24c72c10bece8a80b8680ae915d6c6f68 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Sun, 24 Mar 2024 14:30:18 -0400 Subject: [PATCH 042/119] Handle onDidChangeWatchedFiles directory deletions --- src/ProgramBuilder.ts | 7 ++++-- src/lsp/DocumentManager.ts | 4 ++++ src/lsp/Project.ts | 44 ++++++++++++++++++++++------------ src/lsp/ProjectManager.spec.ts | 37 ++++++++++++++++++++++++---- src/lsp/ProjectManager.ts | 14 ++--------- 5 files changed, 72 insertions(+), 34 deletions(-) diff --git a/src/ProgramBuilder.ts b/src/ProgramBuilder.ts index e11cac07f..1f98481be 100644 --- a/src/ProgramBuilder.ts +++ b/src/ProgramBuilder.ts @@ -534,15 +534,18 @@ export class ProgramBuilder { /** * Remove all files from the program that are in the specified folder path - * @param srcPath the path to the + * @param srcPath the path to the folder to remove */ - public removeFilesInFolder(srcPath: string) { + public removeFilesInFolder(srcPath: string): boolean { + let removedSomeFiles = false; for (let filePath in this.program.files) { //if the file path starts with the parent path and the file path does not exactly match the folder path if (filePath.startsWith(srcPath) && filePath !== srcPath) { this.program.removeFile(filePath); + removedSomeFiles = true; } } + return removedSomeFiles; } /** diff --git a/src/lsp/DocumentManager.ts b/src/lsp/DocumentManager.ts index 7354f9a30..42b14f46e 100644 --- a/src/lsp/DocumentManager.ts +++ b/src/lsp/DocumentManager.ts @@ -37,6 +37,7 @@ export class DocumentManager { srcPath: srcPath, fileContents: fileContents }); + //schedule a future flush this.throttle(); } @@ -56,6 +57,9 @@ export class DocumentManager { } //register this delete this.queue.set(srcPath, { type: 'delete', srcPath: srcPath }); + + //schedule a future flush + this.throttle(); } /** diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 5eb84d0ae..afde4fa82 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -20,7 +20,6 @@ export class Project implements LspProject { * Activates this project. Every call to `activate` should completely reset the project, clear all used ram and start from scratch. */ public async activate(options: ActivateOptions) { - this.activationDeferred = new Deferred(); this.projectPath = options.projectPath; this.workspaceFolder = options.workspaceFolder; @@ -81,6 +80,11 @@ export class Project implements LspProject { this.activationDeferred.resolve(); } + /** + * Gets resolved when the project has finished activating + */ + private activationDeferred = new Deferred(); + /** * Promise that resolves when the project finishes activating * @returns a promise that resolves when the project finishes activating @@ -123,10 +127,6 @@ export class Project implements LspProject { return this.builder.program.options; } - /** - * Gets resolved when the project has finished activating - */ - private activationDeferred: Deferred; public getDiagnostics() { const diagnostics = this.builder.getDiagnostics(); @@ -165,12 +165,13 @@ export class Project implements LspProject { let didChangeFiles = false; for (const action of documentActions) { let didChangeThisFile = false; - if (this.willAcceptFile(action.srcPath)) { - if (action.type === 'set') { - didChangeThisFile = this.setFile(action.srcPath, action.fileContents); - } else if (action.type === 'delete') { - didChangeThisFile = this.removeFile(action.srcPath); - } + //if this is a `set` and the file matches the project's files array, set it + if (action.type === 'set' && this.willAcceptFile(action.srcPath)) { + didChangeThisFile = this.setFile(action.srcPath, action.fileContents); + + //try to delete the file or directory + } else if (action.type === 'delete') { + didChangeThisFile = this.removeFileOrDirectory(action.srcPath); } didChangeFiles = didChangeFiles || didChangeThisFile; } @@ -217,15 +218,28 @@ export class Project implements LspProject { /** * Remove the in-memory file at the specified path. This is typically called when the user (or file system watcher) triggers a file delete * @param srcPath absolute path to the File - * @returns true if we found and removed the file. false if we didn't have a file to remove + * @returns true if we found and removed at least one file, or false if no files were removed */ - private removeFile(srcPath: string) { + private removeFileOrDirectory(srcPath: string) { + srcPath = util.standardizePath(srcPath); + //if this is a direct file match, remove the file if (this.builder.program.hasFile(srcPath)) { this.builder.program.removeFile(srcPath); return true; - } else { - return false; } + + //maybe this is a directory. Remove all files that start with this path + let removedSomeFiles = false; + let lowerSrcPath = srcPath.toLowerCase(); + for (let file of Object.values(this.builder.program.files)) { + //if the file path starts with the parent path and the file path does not exactly match the folder path + if (file.srcPath?.toLowerCase().startsWith(lowerSrcPath)) { + this.builder.program.removeFile(file.srcPath, false); + removedSomeFiles = true; + } + } + //return true if we removed at least one file + return removedSomeFiles; } /** diff --git a/src/lsp/ProjectManager.spec.ts b/src/lsp/ProjectManager.spec.ts index be08f5d5f..bea7df443 100644 --- a/src/lsp/ProjectManager.spec.ts +++ b/src/lsp/ProjectManager.spec.ts @@ -228,6 +228,31 @@ describe('ProjectManager', () => { DiagnosticMessages.cannotFindName('three').message ]); }); + + it('removes all files in a folder', async () => { + fsExtra.outputFileSync(`${rootDir}/source/main.brs`, `sub main():print "main":end sub`); + fsExtra.outputFileSync(`${rootDir}/source/libs/alpha/beta.brs`, `sub beta(): print one: end sub`); + fsExtra.outputFileSync(`${rootDir}/source/libs/alpha/charlie/delta.brs`, `sub delta():print two:end sub`); + fsExtra.outputFileSync(`${rootDir}/source/libs/echo/foxtrot.brs`, `sub foxtrot():print three:end sub`); + + await manager.syncProjects([{ + workspaceFolder: rootDir + }]); + + expectDiagnostics(await onNextDiagnostics(), [ + DiagnosticMessages.cannotFindName('one').message, + DiagnosticMessages.cannotFindName('two').message, + DiagnosticMessages.cannotFindName('three').message + ]); + + await manager.handleFileChanges([ + //register the entire folder as an "add" + { srcPath: `${rootDir}/source/libs`, type: FileChangeType.Deleted } + ]); + + expectZeroDiagnostics(await onNextDiagnostics()); + }); + }); describe('threading', () => { @@ -323,12 +348,14 @@ describe('ProjectManager', () => { }); }); - describe('raceUntil', () => { - beforeEach(() => { - }); - + describe('findFirstMatchingProject', () => { async function doTest(expectedIndex: number, ...values: any[]) { - manager.projects = [{ index: 0 }, { index: 1 }, { index: 2 }] as any; + manager.projects = [0, 1, 2].map((i) => { + const project = new Project(); + project['index'] = i; + project['activationDeferred'].resolve(); + return project; + }); let idx = 0; expect( diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index ca0cc3fb9..df2b77ae7 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -128,16 +128,6 @@ export class ProjectManager { return this.syncPromise; } - /** - * Set new contents for a file. This is safe to call any time. Changes will be queued and flushed at the correct times - * during the program's lifecycle flow - * @param srcPath absolute source path of the file - * @param fileContents the text contents of the file - */ - private setFile(srcPath: string, fileContents: string) { - this.documentManager.set(srcPath, fileContents); - } - /** * Promise that resolves when all file changes have been processed (so we can queue file changes in sequence) */ @@ -176,7 +166,7 @@ export class ProjectManager { //this is a new file. set the file contents if (fsExtra.statSync(srcPath).isFile()) { const fileContents = change.fileContents ?? (await fsExtra.readFile(change.srcPath, 'utf8')).toString(); - this.setFile(change.srcPath, fileContents); + this.documentManager.set(change.srcPath, fileContents); //if this is a new directory, read all files recursively and register those as file changes too } else { @@ -549,7 +539,7 @@ export class ProjectManager { public dispose() { this.emitter.removeAllListeners(); for (const project of this.projects) { - project.dispose(); + project?.dispose?.(); } } } From 727f9aaa2459027faf705105612ce89005ee95e3 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 25 Mar 2024 01:07:08 -0400 Subject: [PATCH 043/119] Reload project when bsconfig changes. Remove getOptions from LspProject interface --- src/DiagnosticCollection.spec.ts | 34 ++++++---- src/DiagnosticCollection.ts | 34 ++++------ src/LanguageServer.ts | 18 ++++-- src/lsp/LspProject.ts | 37 +++++++++-- src/lsp/Project.ts | 34 ++++++---- src/lsp/ProjectManager.ts | 31 ++++++--- src/lsp/worker/WorkerThreadProject.ts | 92 +++++++++++++-------------- 7 files changed, 168 insertions(+), 112 deletions(-) diff --git a/src/DiagnosticCollection.spec.ts b/src/DiagnosticCollection.spec.ts index a5ec31df5..18571bb45 100644 --- a/src/DiagnosticCollection.spec.ts +++ b/src/DiagnosticCollection.spec.ts @@ -1,8 +1,7 @@ import { DiagnosticCollection } from './DiagnosticCollection'; import util from './util'; import { expect } from './chai-config.spec'; -import { Project } from './lsp/Project'; -import type { LspDiagnostic, LspProject } from './lsp/LspProject'; +import type { LspDiagnostic } from './lsp/LspProject'; import { URI } from 'vscode-uri'; import { rootDir } from './testHelpers.spec'; import * as path from 'path'; @@ -11,21 +10,20 @@ import { interpolatedRange } from './astUtils/creators'; describe('DiagnosticCollection', () => { let collection: DiagnosticCollection; - let project: Project; + let projectId: number; beforeEach(() => { collection = new DiagnosticCollection(); - project = new Project(); - project.projectPath = rootDir; + projectId = 1; }); function testPatch(options: { - project?: LspProject; + projectId?: number; diagnosticsByFile?: Record>; expected?: Record; }) { - const patch = collection.getPatch(options.project ?? project, createDiagnostics(options.diagnosticsByFile ?? {})); + const patch = collection.getPatch(options.projectId ?? projectId, createDiagnostics(options.diagnosticsByFile ?? {})); //convert the patch into our test structure const actual = {}; for (let filePath in patch) { @@ -44,12 +42,22 @@ describe('DiagnosticCollection', () => { expect(actual).to.eql(expected); } + it('computes patch for empty diagnostics', () => { + //start with 1 diagnostic + testPatch({ + diagnosticsByFile: { + 'source/file1.brs': ['message1'] + }, + expected: { + 'source/file1.brs': ['message1'] + } + }); + }); + it('computes patch for specific project', () => { - const project1 = new Project(); - const project2 = new Project(); //should be all diagnostics from project1 testPatch({ - project: project1, + projectId: 1, diagnosticsByFile: { 'alpha.brs': ['a1', 'a2'], 'beta.brs': ['b1', 'b2'] @@ -62,7 +70,7 @@ describe('DiagnosticCollection', () => { //set project2 diagnostics that overlap a little with project1 testPatch({ - project: project2, + projectId: 2, diagnosticsByFile: { 'beta.brs': ['b2', 'b3'], 'charlie.brs': ['c1', 'c2'] @@ -76,7 +84,7 @@ describe('DiagnosticCollection', () => { //set project 1 diagnostics again (same diagnostics) testPatch({ - project: project1, + projectId: 1, diagnosticsByFile: { 'alpha.brs': ['a1', 'a2'], 'beta.brs': ['b1', 'b2'] @@ -200,7 +208,7 @@ describe('DiagnosticCollection', () => { message: 'message1', range: interpolatedRange, uri: undefined, - projects: [project] + projects: [projectId] } as any] }, expected: { diff --git a/src/DiagnosticCollection.ts b/src/DiagnosticCollection.ts index a38d8376e..c0ce905cd 100644 --- a/src/DiagnosticCollection.ts +++ b/src/DiagnosticCollection.ts @@ -10,11 +10,9 @@ export class DiagnosticCollection { * Get a patch of any changed diagnostics since last time. This takes a single project and diagnostics, but evaulates * the patch based on all previously seen projects. It's supposed to be a rolling patch. * This will include _ALL_ diagnostics for a file if any diagnostics have changed for that file, due to how the language server expects diagnostics to be sent. - * @param projects - * @returns */ - public getPatch(project: LspProject, diagnostics: LspDiagnostic[]) { - const diagnosticsByFile = this.getDiagnosticsByFile(project, diagnostics as KeyedDiagnostic[]); + public getPatch(projectId: number, diagnostics: LspDiagnostic[]) { + const diagnosticsByFile = this.getDiagnosticsByFile(projectId, diagnostics as KeyedDiagnostic[]); const patch = { ...this.getRemovedPatch(diagnosticsByFile), @@ -29,11 +27,10 @@ export class DiagnosticCollection { /** * Get all the previous diagnostics, remove any that were exclusive to the current project, then mix in the project's new diagnostics. - * @param project the latest project that should have its diagnostics refreshed + * @param projectId the id of the project that should have its diagnostics refreshed * @param thisProjectDiagnostics diagnostics for the project - * @returns */ - private getDiagnosticsByFile(project: LspProject, thisProjectDiagnostics: KeyedDiagnostic[]) { + private getDiagnosticsByFile(projectId: number, thisProjectDiagnostics: KeyedDiagnostic[]) { const result = this.clonePreviousDiagnosticsByFile(); const diagnosticsByKey = new Map(); @@ -47,13 +44,10 @@ export class DiagnosticCollection { //remember this diagnostic key for use when deduping down below diagnosticsByKey.set(diagnostic.key, diagnostic); - const idx = diagnostic.projects.indexOf(project); //unlink the diagnostic from this project - if (idx > -1) { - diagnostic.projects.splice(idx, 1); - } + diagnostic.projectIds.delete(projectId); //delete this diagnostic if it's no longer linked to any projects - if (diagnostic.projects.length === 0) { + if (diagnostic.projectIds.size === 0) { diagnostics.splice(i, 1); diagnosticsByKey.delete(diagnostic.key); } @@ -80,7 +74,7 @@ export class DiagnosticCollection { range.end.character + diagnostic.message; - diagnostic.projects ??= [project]; + diagnostic.projectIds ??= new Set([projectId]); //don't include duplicates if (!diagnosticsByKey.has(diagnostic.key)) { @@ -90,11 +84,8 @@ export class DiagnosticCollection { diagnosticsForFile.push(diagnostic); } - const projects = diagnosticsByKey.get(diagnostic.key).projects; //link this project to the diagnostic - if (!projects.includes(project)) { - projects.push(project); - } + diagnosticsByKey.get(diagnostic.key).projectIds.add(projectId); } //sort the list so it's easier to compare later @@ -115,7 +106,7 @@ export class DiagnosticCollection { clone[key].push({ ...diagnostic, //make a copy of the projects array (but keep the project references intact) - projects: [...diagnostic.projects] + projectIds: new Set([...diagnostic.projectIds]) }); } } @@ -128,7 +119,8 @@ export class DiagnosticCollection { private getRemovedPatch(currentDiagnosticsByFile: Record) { const result = {} as Record; for (const filePath in this.previousDiagnosticsByFile) { - if (!currentDiagnosticsByFile[filePath]) { + //if there are no current diagnostics for this file, add an empty array to the result for that file path + if ((currentDiagnosticsByFile[filePath]?.length ?? 0) === 0) { result[filePath] = []; } } @@ -173,7 +165,7 @@ export class DiagnosticCollection { private getAddedPatch(currentDiagnosticsByFile: Record) { const result = {} as Record; for (const filePath in currentDiagnosticsByFile) { - if (!this.previousDiagnosticsByFile[filePath]) { + if ((this.previousDiagnosticsByFile[filePath]?.length ?? 0) === 0) { result[filePath] = currentDiagnosticsByFile[filePath]; } } @@ -183,5 +175,5 @@ export class DiagnosticCollection { interface KeyedDiagnostic extends LspDiagnostic { key: string; - projects: LspProject[]; + projectIds: Set; } diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index fa02e7a7e..58f6e9679 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -39,9 +39,7 @@ import { ProposedFeatures, TextDocuments, TextDocumentSyncKind, - CodeActionKind, - DidChangeWatchedFilesNotification, - WatchKind + CodeActionKind } from 'vscode-languageserver/node'; import { URI } from 'vscode-uri'; import { TextDocument } from 'vscode-languageserver-textdocument'; @@ -109,6 +107,17 @@ export class LanguageServer implements OnHandler { void this.sendDiagnostics(event); }); + // Send all open document changes whenever a project reloads. This is necessary because the project loads files from disk + // and may not have the latest unsaved file changes. Any existing projects that already use these files will just ignore the changes + // because the file contents haven't changed. + this.projectManager.on('project-reload', (event) => { + for (const document of this.documents.all()) { + void this.onTextDocumentDidChangeContent({ + document: document + }); + } + }); + this.projectManager.busyStatusTracker.on('change', (event) => { this.sendBusyStatus(); }); @@ -417,7 +426,6 @@ export class LanguageServer implements OnHandler { @AddStackToErrorMessage public async onExecuteCommand(params: ExecuteCommandParams) { - await this.waitAllProjectFirstRuns(); if (params.command === CustomCommands.TranspileFile) { const result = await this.projectManager.transpileFile(params.arguments[0]); //back-compat: include `pathAbsolute` property so older vscode versions still work @@ -838,7 +846,7 @@ export class LanguageServer implements OnHandler { * Send diagnostics to the client */ private async sendDiagnostics(options: { project: LspProject; diagnostics: LspDiagnostic[] }) { - const patch = this.diagnosticCollection.getPatch(options.project, options.diagnostics); + const patch = this.diagnosticCollection.getPatch(options.project.projectNumber, options.diagnostics); await Promise.all(Object.keys(patch).map(async (srcPath) => { const uri = URI.file(srcPath).toString(); diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index 4320e4457..a1dc7f754 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,14 +1,19 @@ import type { Diagnostic, Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction, CompletionList } from 'vscode-languageserver'; import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; -import type { BsConfig } from '../BsConfig'; import type { DocumentAction } from './DocumentManager'; import type { FileTranspileResult, SignatureInfoObj } from '../Program'; +import type { ProjectConfig } from './ProjectManager'; /** * Defines the contract between the ProjectManager and the main or worker thread Project classes */ export interface LspProject { + /** + * The config used to initialize this project. Only here to use when reloading the project + */ + projectConfig: ProjectConfig; + /** * The path to where the project resides */ @@ -19,11 +24,23 @@ export interface LspProject { */ projectNumber: number; + /** + * The root directory of the project. + * Only available after `.activate()` has completed + */ + rootDir: string; + + /** + * Path to a bsconfig.json file that will be used for this project. + * Only available after `.activate()` has completed + */ + configFilePath?: string; + /** * Initialize and start running the project. This will scan for all files, and build a full project in memory, then validate the project * @param options */ - activate(options: ActivateOptions): MaybePromise; + activate(options: ActivateOptions): MaybePromise; /** * Get a promise that resolves when the project finishes activating @@ -41,11 +58,6 @@ export interface LspProject { */ cancelValidate(): MaybePromise; - /** - * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. - */ - getOptions(): MaybePromise; - /** * Get the list of all file paths that are currently loaded in the project */ @@ -159,3 +171,14 @@ export interface ActivateOptions { export interface LspDiagnostic extends Diagnostic { uri: string; } + +export interface ActivateResponse { + /** + * The root directory of the project + */ + rootDir: string; + /** + * The path to the config file (i.e. `bsconfig.json`) that was used to load this project + */ + configFilePath: string; +} diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index afde4fa82..649281485 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -2,7 +2,7 @@ import { ProgramBuilder } from '../ProgramBuilder'; import * as EventEmitter from 'eventemitter3'; import util, { standardizePath as s } from '../util'; import * as path from 'path'; -import type { ActivateOptions, LspDiagnostic, LspProject } from './LspProject'; +import type { ActivateOptions, ActivateResponse, LspDiagnostic, LspProject } from './LspProject'; import type { CompilerPlugin, Hover, MaybePromise } from '../interfaces'; import { DiagnosticMessages } from '../DiagnosticMessages'; import { URI } from 'vscode-uri'; @@ -13,14 +13,22 @@ import { CompletionList } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; import type { DocumentAction } from './DocumentManager'; import type { SignatureInfoObj } from '../Program'; +import type { ProjectConfig } from './ProjectManager'; export class Project implements LspProject { + constructor( + /** + * The config used to create this project. Mostly just here to use when reloading this project + */ + public projectConfig: ProjectConfig + ) { + + } /** * Activates this project. Every call to `activate` should completely reset the project, clear all used ram and start from scratch. */ - public async activate(options: ActivateOptions) { - + public async activate(options: ActivateOptions): Promise { this.projectPath = options.projectPath; this.workspaceFolder = options.workspaceFolder; this.projectNumber = options.projectNumber; @@ -78,6 +86,15 @@ export class Project implements LspProject { void this.validate(); this.activationDeferred.resolve(); + + return { + configFilePath: this.configFilePath, + rootDir: this.builder.program.options.rootDir + }; + } + + public get rootDir() { + return this.builder.program.options.rootDir; } /** @@ -120,14 +137,6 @@ export class Project implements LspProject { delete this.validationCancelToken; } - /** - * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. - */ - public getOptions() { - return this.builder.program.options; - } - - public getDiagnostics() { const diagnostics = this.builder.getDiagnostics(); return diagnostics.map(x => { @@ -162,6 +171,7 @@ export class Project implements LspProject { * This will cancel any pending validation cycles and queue a future validation cycle instead. */ public async applyFileChanges(documentActions: DocumentAction[]): Promise { + await this.onIdle(); let didChangeFiles = false; for (const action of documentActions) { let didChangeThisFile = false; @@ -408,7 +418,7 @@ export class Project implements LspProject { public dispose() { this.builder?.dispose(); - this.emitter.removeAllListeners(); + this.emitter?.removeAllListeners(); if (this.activationDeferred?.isCompleted === false) { this.activationDeferred.reject( new Error('Project was disposed, activation has been aborted') diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index df2b77ae7..d9bd5818c 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -40,6 +40,7 @@ export class ProjectManager { * @param event the document changes that have occurred since the last time we applied */ @TrackBusyStatus + @OnReady private async applyDocumentChanges(event: FlushEvent) { //apply all of the document actions to each project in parallel await Promise.all(this.projects.map(async (project) => { @@ -184,6 +185,21 @@ export class ProjectManager { })); } } + + //reload any projects whose bsconfig.json was changed + const projectsToReload = this.projects.filter(x => x.configFilePath?.toLowerCase() === change.srcPath.toLowerCase()); + await Promise.all( + projectsToReload.map(x => this.reloadProject(x)) + ); + } + + /** + * Given a project, forcibly reload it by removing it and re-adding it + */ + private async reloadProject(project: LspProject) { + this.removeProject(project); + await this.createProject(project.projectConfig); + this.emit('project-reload', { project: project }); } /** @@ -498,8 +514,8 @@ export class ProjectManager { } let project: LspProject = config.threadingEnabled - ? new WorkerThreadProject() - : new Project(); + ? new WorkerThreadProject(config) + : new Project(config); this.projects.push(project); @@ -510,15 +526,13 @@ export class ProjectManager { project: project } as any); }); + config.projectNumber ??= ProjectManager.projectNumberSequence++; - await project.activate({ - projectPath: config.projectPath, - workspaceFolder: config.workspaceFolder, - projectNumber: config.projectNumber ?? ProjectManager.projectNumberSequence++ - }); + await project.activate(config); } public on(eventName: 'critical-failure', handler: (data: { project: LspProject; message: string }) => MaybePromise); + public on(eventName: 'project-reload', handler: (data: { project: LspProject }) => MaybePromise); public on(eventName: 'diagnostics', handler: (data: { project: LspProject; diagnostics: LspDiagnostic[] }) => MaybePromise); public on(eventName: string, handler: (payload: any) => MaybePromise) { this.emitter.on(eventName, handler as any); @@ -528,6 +542,7 @@ export class ProjectManager { } private emit(eventName: 'critical-failure', data: { project: LspProject; message: string }); + private emit(eventName: 'project-reload', data: { project: LspProject }); private emit(eventName: 'diagnostics', data: { project: LspProject; diagnostics: LspDiagnostic[] }); private async emit(eventName: string, data?) { //emit these events on next tick, otherwise they will be processed immediately which could cause issues @@ -564,7 +579,7 @@ export interface WorkspaceConfig { threadingEnabled?: boolean; } -interface ProjectConfig { +export interface ProjectConfig { /** * Path to the project */ diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index 26c3dee25..d0fa79ab0 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -3,17 +3,17 @@ import { Worker } from 'worker_threads'; import type { WorkerMessage } from './MessageHandler'; import { MessageHandler } from './MessageHandler'; import util from '../../util'; -import type { LspDiagnostic } from '../LspProject'; +import type { LspDiagnostic, ActivateResponse } from '../LspProject'; import { type ActivateOptions, type LspProject } from '../LspProject'; import { isMainThread, parentPort } from 'worker_threads'; import { WorkerThreadProjectRunner } from './WorkerThreadProjectRunner'; import { WorkerPool } from './WorkerPool'; import type { Hover, MaybePromise, SemanticToken } from '../../interfaces'; -import type { BsConfig } from '../../BsConfig'; import type { DocumentAction } from '../DocumentManager'; import { Deferred } from '../../deferred'; import type { FileTranspileResult, SignatureInfoObj } from '../../Program'; import type { Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction, CompletionList } from 'vscode-languageserver-protocol'; +import type { ProjectConfig } from '../ProjectManager'; export const workerPool = new WorkerPool(() => { return new Worker( @@ -37,6 +37,15 @@ if (!isMainThread) { export class WorkerThreadProject implements LspProject { + constructor( + /** + * The config used to create this project. Mostly just here to use when reloading this project + */ + public projectConfig: ProjectConfig + ) { + + } + public async activate(options: ActivateOptions) { this.projectPath = options.projectPath; this.workspaceFolder = options.workspaceFolder; @@ -52,17 +61,50 @@ export class WorkerThreadProject implements LspProject { onUpdate: this.processUpdate.bind(this) }); - await this.messageHandler.sendRequest('activate', { data: [options] }); + const activateResponse = await this.messageHandler.sendRequest('activate', { data: [options] }); + this.configFilePath = activateResponse.data.configFilePath; + this.rootDir = activateResponse.data.rootDir; //populate a few properties with data from the thread so we can use them for some synchronous checks this.filePaths = new Set(await this.getFilePaths()); - this.options = await this.getOptions(); this.activationDeferred.resolve(); + return activateResponse.data; } private activationDeferred = new Deferred(); + /** + * The root directory of the project + */ + public rootDir: string; + + /** + * Path to a bsconfig.json file that will be used for this project + */ + public configFilePath?: string; + + /** + * The worker thread where the actual project will execute + */ + private worker: Worker; + + /** + * The path to where the project resides + */ + public projectPath: string; + + /** + * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging + */ + public projectNumber: number; + + /** + * The path to the workspace where this project resides. A workspace can have multiple projects (by adding a bsconfig.json to each folder). + * Defaults to `.projectPath` if not set + */ + public workspaceFolder: string; + /** * Promise that resolves when the project finishes activating * @returns a promise that resolves when the project finishes activating @@ -127,22 +169,6 @@ export class WorkerThreadProject implements LspProject { return (await this.messageHandler.sendRequest('getFilePaths')).data; } - /** - * Get the bsconfig options from the program. Should only be called after `.activate()` has completed. - */ - public async getOptions() { - return (await this.messageHandler.sendRequest('getOptions')).data; - } - - /** - * A local reference to the bsconfig this project was built with. Should only be accessed after `.activate()` has completed. - */ - private options: BsConfig; - - public get rootDir() { - return this.options.rootDir; - } - /** * Send a request with the standard structure * @param name the name of the request @@ -213,32 +239,6 @@ export class WorkerThreadProject implements LspProject { this.emit(update.name as any, update.data); } - /** - * The worker thread where the actual project will execute - */ - private worker: Worker; - - /** - * The path to where the project resides - */ - public projectPath: string; - - /** - * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging - */ - public projectNumber: number; - - /** - * The path to the workspace where this project resides. A workspace can have multiple projects (by adding a bsconfig.json to each folder). - * Defaults to `.projectPath` if not set - */ - public workspaceFolder: string; - - /** - * Path to a bsconfig.json file that will be used for this project - */ - public configFilePath?: string; - public on(eventName: 'critical-failure', handler: (data: { message: string }) => void); public on(eventName: 'diagnostics', handler: (data: { diagnostics: LspDiagnostic[] }) => MaybePromise); public on(eventName: 'all', handler: (eventName: string, data: any) => MaybePromise); From 9bf41b37cde1d111e30457802c6642bd5dd685c5 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 25 Mar 2024 08:31:07 -0400 Subject: [PATCH 044/119] Remove document file resolver (we handle that a different way now) --- src/LanguageServer.ts | 19 ------------------- src/lsp/Project.spec.ts | 2 +- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index 58f6e9679..cbcb23794 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -122,10 +122,6 @@ export class LanguageServer implements OnHandler { this.sendBusyStatus(); }); - //allow the lsp to provide file contents - //TODO handle this... - // this.projectManager.addFileResolver(this.documentFileResolver.bind(this)); - // Create a connection for the server. The connection uses Node's IPC as a transport. this.establishConnection(); @@ -539,18 +535,6 @@ export class LanguageServer implements OnHandler { void this.connection.sendNotification('critical-failure', message); } - /** - * Event handler for when the program wants to load file contents. - * anytime the program wants to load a file, check with our in-memory document cache first - */ - private documentFileResolver(srcPath: string) { - let pathUri = URI.file(srcPath).toString(); - let document = this.documents.get(pathUri); - if (document) { - return document.getText(); - } - } - private async createStandaloneFileProject(srcPath: string) { //skip this workspace if we already have it if (this.standaloneFileProjects[srcPath]) { @@ -562,9 +546,6 @@ export class LanguageServer implements OnHandler { //prevent clearing the console on run...this isn't the CLI so we want to keep a full log of everything builder.allowConsoleClearing = false; - //look for files in our in-memory cache before going to the file system - builder.addFileResolver(this.documentFileResolver.bind(this)); - //get the path to the directory where this file resides let cwd = path.dirname(srcPath); diff --git a/src/lsp/Project.spec.ts b/src/lsp/Project.spec.ts index b0bcf6897..bb8766a23 100644 --- a/src/lsp/Project.spec.ts +++ b/src/lsp/Project.spec.ts @@ -13,7 +13,7 @@ describe('Project', () => { beforeEach(() => { sinon.restore(); - project = new Project(); + project = new Project({} as any); fsExtra.emptyDirSync(tempDir); }); From 3226d791f5e6f772c131c21029b12a38b158d9f8 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Mon, 25 Mar 2024 10:45:10 -0400 Subject: [PATCH 045/119] Skip updating changed files if their contents haven't changed --- src/lsp/Project.spec.ts | 32 ++++++++++++++++++++++++++++++++ src/lsp/Project.ts | 8 +++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/lsp/Project.spec.ts b/src/lsp/Project.spec.ts index bb8766a23..66e7fa940 100644 --- a/src/lsp/Project.spec.ts +++ b/src/lsp/Project.spec.ts @@ -40,6 +40,38 @@ describe('Project', () => { }); }); + describe('setFile', () => { + it('skips setting the file if the contents have not changed', async () => { + await project.activate({ projectPath: rootDir }); + //initial set should be true + expect( + await project.applyFileChanges([{ + fileContents: 'sub main:end sub', + srcPath: s`${rootDir}/source/main.brs`, + type: 'set' + }]) + ).to.be.true; + + //contents haven't changed, this should be false + expect( + await project.applyFileChanges([{ + fileContents: 'sub main:end sub', + srcPath: s`${rootDir}/source/main.brs`, + type: 'set' + }]) + ).to.be.false; + + //contents changed again, should be true + expect( + await project.applyFileChanges([{ + fileContents: 'sub main2:end sub', + srcPath: s`${rootDir}/source/main.brs`, + type: 'set' + }]) + ).to.be.true; + }); + }); + describe('activate', () => { it('finds bsconfig.json at root', async () => { fsExtra.outputFileSync(`${rootDir}/bsconfig.json`, ''); diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 649281485..77c433523 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -203,7 +203,7 @@ export class Project implements LspProject { * during the program's lifecycle flow * @param srcPath absolute source path of the file * @param fileContents the text contents of the file - * @returns true if this program accepted and added the file. false if this file doesn't match against the program's files array + * @returns true if this program accepted and added the file. false if the program didn't want the file, or if the contents didn't change */ private setFile(srcPath: string, fileContents: string) { const { files, rootDir } = this.builder.program.options; @@ -211,6 +211,12 @@ export class Project implements LspProject { //get the dest path for this file. let destPath = rokuDeploy.getDestPath(srcPath, files, rootDir); + //if we have a file and the contents haven't changed + let file = this.builder.program.getFile(destPath); + if (file && file.fileContents === fileContents) { + return false; + } + //if we got a dest path, then the program wants this file if (destPath) { this.builder.program.setFile( From 629f52adbe4b00a374336e27217800171ada9331 Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 26 Mar 2024 13:41:18 -0400 Subject: [PATCH 046/119] First try at implementing standalone workspaces (broken) --- src/LanguageServer.ts | 311 ++++---------------------- src/interfaces.ts | 4 + src/lsp/DocumentManager.ts | 7 +- src/lsp/LspProject.ts | 13 +- src/lsp/Project.ts | 19 +- src/lsp/ProjectManager.ts | 61 ++++- src/lsp/worker/WorkerThreadProject.ts | 12 +- 7 files changed, 124 insertions(+), 303 deletions(-) diff --git a/src/LanguageServer.ts b/src/LanguageServer.ts index cbcb23794..c3006686e 100644 --- a/src/LanguageServer.ts +++ b/src/LanguageServer.ts @@ -47,7 +47,6 @@ import type { BsConfig } from './BsConfig'; import { ProgramBuilder } from './ProgramBuilder'; import { standardizePath as s, util } from './util'; import { Logger } from './Logger'; -import { Throttler } from './Throttler'; import { DiagnosticCollection } from './DiagnosticCollection'; import { encodeSemanticTokens, semanticTokensLegend } from './SemanticTokenUtils'; import type { WorkspaceConfig } from './lsp/ProjectManager'; @@ -90,14 +89,6 @@ export class LanguageServer implements OnHandler { private loggerSubscription: () => void; - public validateThrottler = new Throttler(0); - - private boundValidateAll = this.validateAll.bind(this); - - private validateAllThrottled() { - return this.validateThrottler.run(this.boundValidateAll); - } - //run the server public run() { this.projectManager = new ProjectManager(); @@ -242,20 +233,14 @@ export class LanguageServer implements OnHandler { } } - /** - * Provide a list of completion items based on the current cursor position - */ @AddStackToErrorMessage - public async onCompletion(params: CompletionParams, token: CancellationToken, workDoneProgress: WorkDoneProgressReporter, resultProgress: ResultProgressReporter): Promise { - const srcPath = util.uriToPath(params.textDocument.uri); - const completions = await this.projectManager.getCompletions({ srcPath: srcPath, position: params.position }); - return completions; - } - - @AddStackToErrorMessage - protected async onDidChangeConfiguration(args: DidChangeConfigurationParams) { - //if the user changes any user/workspace config settings, just mass-reload all projects - await this.syncProjects(true); + private async onTextDocumentDidChangeContent(event: TextDocumentChangeEvent) { + await this.projectManager.handleFileChanges([{ + srcPath: URI.parse(event.document.uri).fsPath, + type: FileChangeType.Changed, + fileContents: event.document.getText(), + allowStandaloneProject: true + }]); } /** @@ -269,94 +254,41 @@ export class LanguageServer implements OnHandler { await this.projectManager.handleFileChanges( params.changes.map(x => ({ srcPath: util.uriToPath(x.uri), - type: x.type + type: x.type, + //if this is an open document, allow this file to be loaded in a standalone project (if applicable) + allowStandaloneProject: this.documents.get(x.uri) !== undefined })) ); - return; - - let projects = this.getProjects(); - - //convert all file paths to absolute paths - let changes = params.changes.map(x => { - return { - type: x.type, - srcPath: s`${URI.parse(x.uri).fsPath}` - }; - }); - - let keys = changes.map(x => x.srcPath); - - //filter the list of changes to only the ones that made it through the debounce unscathed - changes = changes.filter(x => keys.includes(x.srcPath)); - - //if we have changes to work with - if (changes.length > 0) { - - //if any bsconfig files were added or deleted, re-sync all projects instead of the more specific approach below - if (changes.find(x => (x.type === FileChangeType.Created || x.type === FileChangeType.Deleted) && path.basename(x.srcPath).toLowerCase() === 'bsconfig.json')) { - return this.syncProjects(); - } - - //reload any workspace whose bsconfig.json file has changed - { - let projectsToReload = [] as Project[]; - //get the file paths as a string array - let filePaths = changes.map((x) => x.srcPath); - - for (let project of projects) { - if (project.configFilePath && filePaths.includes(project.configFilePath)) { - projectsToReload.push(project); - } - } - if (projectsToReload.length > 0) { - //vsc can generate a ton of these changes, for vsc system files, so we need to bail if there's no work to do on any of our actual project files - //reload any projects that need to be reloaded - await this.reloadProjects(projectsToReload); - } - - //reassign `projects` to the non-reloaded projects - projects = projects.filter(x => !projectsToReload.includes(x)); - } + } - //convert created folders into a list of files of their contents - const directoryChanges = changes - //get only creation items - .filter(change => change.type === FileChangeType.Created) - //keep only the directories - .filter(change => util.isDirectorySync(change.srcPath)); - - //remove the created directories from the changes array (we will add back each of their files next) - changes = changes.filter(x => !directoryChanges.includes(x)); - - //look up every file in each of the newly added directories - const newFileChanges = directoryChanges - //take just the path - .map(x => x.srcPath) - //exclude the roku deploy staging folder - .filter(dirPath => !dirPath.includes('.roku-deploy-staging')) - //get the files for each folder recursively - .flatMap(dirPath => { - //look up all files - let files = fastGlob.sync('**/*', { - absolute: true, - cwd: rokuDeployUtil.toForwardSlashes(dirPath) - }); - return files.map(x => { - return { - type: FileChangeType.Created, - srcPath: s`${x}` - }; - }); - }); + @AddStackToErrorMessage + private async onDocumentClose(event: TextDocumentChangeEvent): Promise { + const { document } = event; + let filePath = URI.parse(document.uri).fsPath; + let standaloneFileProject = this.standaloneFileProjects[filePath]; + //if this was a temp file, close it + if (standaloneFileProject) { + await standaloneFileProject.firstRunPromise; + standaloneFileProject.builder.dispose(); + delete this.standaloneFileProjects[filePath]; + await this.sendDiagnostics(); + } + } - //add the new file changes to the changes array. - changes.push(...newFileChanges as any); + /** + * Provide a list of completion items based on the current cursor position + */ + @AddStackToErrorMessage + public async onCompletion(params: CompletionParams, token: CancellationToken, workDoneProgress: WorkDoneProgressReporter, resultProgress: ResultProgressReporter): Promise { + const srcPath = util.uriToPath(params.textDocument.uri); + const completions = await this.projectManager.getCompletions({ srcPath: srcPath, position: params.position }); + return completions; + } - //give every workspace the chance to handle file changes - await Promise.all( - projects.map((project) => this.handleFileChanges(project, changes)) - ); - } + @AddStackToErrorMessage + protected async onDidChangeConfiguration(args: DidChangeConfigurationParams) { + //if the user changes any user/workspace config settings, just mass-reload all projects + await this.syncProjects(true); } @AddStackToErrorMessage @@ -495,7 +427,7 @@ export class LanguageServer implements OnHandler { excludePatterns: await this.getWorkspaceExcludeGlobs(workspaceFolder), bsconfigPath: config.configFile, //TODO we need to solidify the actual name of this flag in user/workspace settings - threadingEnabled: config.languageServer.enableThreading + threadingEnabled: config.languageServer.enableThreading ?? true } as WorkspaceConfig; }) @@ -603,41 +535,6 @@ export class LanguageServer implements OnHandler { } - /** - * Reload each of the specified workspaces - */ - private async reloadProjects(projects: Project[]) { - await Promise.all( - projects.map(async (project) => { - //ensure the workspace has finished starting up - try { - await project.firstRunPromise; - } catch (e) { } - - //handle standard workspace - if (project.isStandaloneFileProject === false) { - this.removeProject(project); - - //create a new workspace/brs program - await this.createProject(project.projectPath, project.workspacePath, project.projectNumber); - - //handle temp workspace - } else { - project.builder.dispose(); - delete this.standaloneFileProjects[project.projectPath]; - await this.createStandaloneFileProject(project.projectPath); - } - }) - ); - if (projects.length > 0) { - //wait for all of the programs to finish starting up - await this.waitAllProjectFirstRuns(); - - // valdiate all workspaces - this.validateAllThrottled(); //eslint-disable-line - } - } - private getRootDir(workspace: Project) { let options = workspace?.builder?.program?.options; return options?.rootDir ?? options?.cwd; @@ -693,136 +590,6 @@ export class LanguageServer implements OnHandler { } } - /** - * This only operates on files that match the specified files globs, so it is safe to throw - * any file changes you receive with no unexpected side-effects - */ - public async handleFileChanges(project: Project, changes: { type: FileChangeType; srcPath: string }[]) { - //this loop assumes paths are both file paths and folder paths, which eliminates the need to detect. - //All functions below can handle being given a file path AND a folder path, and will only operate on the one they are looking for - let consumeCount = 0; - await Promise.all(changes.map(async (change) => { - consumeCount += await this.handleFileChange(project, change) ? 1 : 0; - })); - - if (consumeCount > 0) { - await this.validateAllThrottled(); - } - } - - /** - * This only operates on files that match the specified files globs, so it is safe to throw - * any file changes you receive with no unexpected side-effects - */ - private async handleFileChange(project: Project, change: { type: FileChangeType; srcPath: string }) { - const { program, options, rootDir } = project.builder; - - //deleted - if (change.type === FileChangeType.Deleted) { - //try to act on this path as a directory - project.builder.removeFilesInFolder(change.srcPath); - - //if this is a file loaded in the program, remove it - if (program.hasFile(change.srcPath)) { - program.removeFile(change.srcPath); - return true; - } else { - return false; - } - - //created - } else if (change.type === FileChangeType.Created) { - // thanks to `onDidChangeWatchedFiles`, we can safely assume that all "Created" changes are file paths, (not directories) - - //get the dest path for this file. - let destPath = rokuDeploy.getDestPath(change.srcPath, options.files, rootDir); - - //if we got a dest path, then the program wants this file - if (destPath) { - program.setFile( - { - src: change.srcPath, - dest: rokuDeploy.getDestPath(change.srcPath, options.files, rootDir) - }, - await project.builder.getFileContents(change.srcPath) - ); - return true; - } else { - //no dest path means the program doesn't want this file - return false; - } - - //changed - } else if (program.hasFile(change.srcPath)) { - //sometimes "changed" events are emitted on files that were actually deleted, - //so determine file existance and act accordingly - if (await util.pathExists(change.srcPath)) { - program.setFile( - { - src: change.srcPath, - dest: rokuDeploy.getDestPath(change.srcPath, options.files, rootDir) - }, - await project.builder.getFileContents(change.srcPath) - ); - } else { - program.removeFile(change.srcPath); - } - return true; - } - } - - @AddStackToErrorMessage - private async onDocumentClose(event: TextDocumentChangeEvent): Promise { - const { document } = event; - let filePath = URI.parse(document.uri).fsPath; - let standaloneFileProject = this.standaloneFileProjects[filePath]; - //if this was a temp file, close it - if (standaloneFileProject) { - await standaloneFileProject.firstRunPromise; - standaloneFileProject.builder.dispose(); - delete this.standaloneFileProjects[filePath]; - await this.sendDiagnostics(); - } - } - - @AddStackToErrorMessage - private async onTextDocumentDidChangeContent(event: TextDocumentChangeEvent) { - await this.projectManager.handleFileChanges([{ - srcPath: URI.parse(event.document.uri).fsPath, - type: FileChangeType.Changed, - fileContents: event.document.getText() - }]); - } - - private async validateAll() { - try { - //synchronize parsing for open files that were included/excluded from projects - await this.synchronizeStandaloneProjects(); - - let projects = this.getProjects(); - - //validate all programs - await Promise.all( - projects.map((project) => { - project.builder.program.validate(); - return project; - }) - ); - } catch (e: any) { - this.connection.console.error(e); - await this.sendCriticalFailure(`Critical error validating project: ${e.message}${e.stack ?? ''}`); - } - } - - private onValidateSettled() { - return Promise.all([ - //wait for the validator to start running (or timeout if it never did) - this.validateThrottler.onRunOnce(100), - //wait for the validator to stop running (or resolve immediately if it's already idle) - this.validateThrottler.onIdleOnce(true) - ]); - } - /** * Send diagnostics to the client */ @@ -848,7 +615,7 @@ export class LanguageServer implements OnHandler { public dispose() { this.loggerSubscription?.(); - this.validateThrottler.dispose(); + this.projectManager?.dispose?.(); } } diff --git a/src/interfaces.ts b/src/interfaces.ts index 17bfc5f78..fde4b886c 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -558,4 +558,8 @@ export interface FileChange { * If provided, this is the new contents of the file. If not provided, the file will be read from disk */ fileContents?: string; + /** + * If true, this file change can have a project created exclusively for it, it no other projects handled it + */ + allowStandaloneProject: boolean; } diff --git a/src/lsp/DocumentManager.ts b/src/lsp/DocumentManager.ts index 42b14f46e..962c7994f 100644 --- a/src/lsp/DocumentManager.ts +++ b/src/lsp/DocumentManager.ts @@ -27,7 +27,7 @@ export class DocumentManager { /** * Add/set the contents of a file */ - public set(srcPath: string, fileContents: string) { + public set(srcPath: string, fileContents: string, allowStandaloneProject: boolean) { srcPath = util.standardizePath(srcPath); if (this.queue.has(srcPath)) { this.queue.delete(srcPath); @@ -35,7 +35,8 @@ export class DocumentManager { this.queue.set(srcPath, { type: 'set', srcPath: srcPath, - fileContents: fileContents + fileContents: fileContents, + allowStandaloneProject: allowStandaloneProject }); //schedule a future flush this.throttle(); @@ -122,6 +123,7 @@ export class DocumentManager { export interface SetDocumentAction { type: 'set'; + allowStandaloneProject: boolean; srcPath: string; fileContents: string; } @@ -131,6 +133,7 @@ export interface DeleteDocumentAction { } export type DocumentAction = SetDocumentAction | DeleteDocumentAction; +export type DocumentActionWithStatus = DocumentAction & { status: 'accepted' | 'rejected' }; export interface FlushEvent { actions: DocumentAction[]; diff --git a/src/lsp/LspProject.ts b/src/lsp/LspProject.ts index a1dc7f754..9703ae8f9 100644 --- a/src/lsp/LspProject.ts +++ b/src/lsp/LspProject.ts @@ -1,6 +1,6 @@ import type { Diagnostic, Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction, CompletionList } from 'vscode-languageserver'; import type { Hover, MaybePromise, SemanticToken } from '../interfaces'; -import type { DocumentAction } from './DocumentManager'; +import type { DocumentAction, DocumentActionWithStatus } from './DocumentManager'; import type { FileTranspileResult, SignatureInfoObj } from '../Program'; import type { ProjectConfig } from './ProjectManager'; @@ -133,7 +133,7 @@ export interface LspProject { * @param documentActions * @returns a boolean indicating whether this project accepted any of the file changes. If false, then this project didn't recognize any of the files and thus did nothing */ - applyFileChanges(documentActions: DocumentAction[]): Promise; + applyFileChanges(documentActions: DocumentAction[]): Promise; /** * An event that is emitted anytime the diagnostics for the project have changed (typically after a validate cycle has finished) @@ -166,6 +166,15 @@ export interface ActivateOptions { * A unique number for this project, generated during this current language server session. Mostly used so we can identify which project is doing logging */ projectNumber?: number; + /** + * If present, this will override any files array found in bsconfig or the default. + * + * The list of file globs used to find all files for the project + * If using the {src;dest;} format, you can specify a different destination directory + * for the matched files in src. + * + */ + files?: Array; } export interface LspDiagnostic extends Diagnostic { diff --git a/src/lsp/Project.ts b/src/lsp/Project.ts index 77c433523..9af5649b7 100644 --- a/src/lsp/Project.ts +++ b/src/lsp/Project.ts @@ -11,7 +11,7 @@ import { rokuDeploy } from 'roku-deploy'; import type { CodeAction, DocumentSymbol, Position, Range, Location, WorkspaceSymbol } from 'vscode-languageserver-protocol'; import { CompletionList } from 'vscode-languageserver-protocol'; import { CancellationTokenSource } from 'vscode-languageserver-protocol'; -import type { DocumentAction } from './DocumentManager'; +import type { DocumentAction, DocumentActionWithStatus } from './DocumentManager'; import type { SignatureInfoObj } from '../Program'; import type { ProjectConfig } from './ProjectManager'; @@ -59,13 +59,11 @@ export class Project implements LspProject { } } as CompilerPlugin); - //register any external file resolvers - //TODO handle in-memory file stuff - // builder.addFileResolver(...this.fileResolvers); - await this.builder.run({ cwd: cwd, project: this.configFilePath, + //if we were given a files array, use it (mostly used for standalone projects) + files: options.files, watch: false, createPackage: false, deploy: false, @@ -170,25 +168,30 @@ export class Project implements LspProject { * Add or replace the in-memory contents of the file at the specified path. This is typically called as the user is typing. * This will cancel any pending validation cycles and queue a future validation cycle instead. */ - public async applyFileChanges(documentActions: DocumentAction[]): Promise { + public async applyFileChanges(documentActions: DocumentAction[]): Promise { await this.onIdle(); let didChangeFiles = false; - for (const action of documentActions) { + const result = documentActions as DocumentActionWithStatus[]; + for (const action of result) { let didChangeThisFile = false; //if this is a `set` and the file matches the project's files array, set it if (action.type === 'set' && this.willAcceptFile(action.srcPath)) { didChangeThisFile = this.setFile(action.srcPath, action.fileContents); + //this file was accepted by the program + action.status = 'accepted'; //try to delete the file or directory } else if (action.type === 'delete') { didChangeThisFile = this.removeFileOrDirectory(action.srcPath); + //if we deleted at least one file, mark this action as accepted + action.status = didChangeThisFile ? 'accepted' : 'rejected'; } didChangeFiles = didChangeFiles || didChangeThisFile; } if (didChangeFiles) { await this.validate(); } - return didChangeFiles; + return result; } /** diff --git a/src/lsp/ProjectManager.ts b/src/lsp/ProjectManager.ts index d9bd5818c..64fb930b8 100644 --- a/src/lsp/ProjectManager.ts +++ b/src/lsp/ProjectManager.ts @@ -43,9 +43,36 @@ export class ProjectManager { @OnReady private async applyDocumentChanges(event: FlushEvent) { //apply all of the document actions to each project in parallel - await Promise.all(this.projects.map(async (project) => { - await project.applyFileChanges(event.actions); + const responses = await Promise.all(this.projects.map(async (project) => { + return project.applyFileChanges(event.actions); })); + + //find actions not handled by any project + for (let i = 0; i < event.actions.length; i++) { + const action = event.actions[i]; + const handledCount = responses.map(x => x[i]).filter(x => x.status === 'accepted').length; + //if this action was handled by zero projects, it's not a delete, and it supports running in a standalone project, then create a a project for it + if (handledCount === 0 && action.type !== 'delete' && action.allowStandaloneProject === true) { + await this.createStandaloneProject(action.srcPath); + } + } + } + + /** + * Create a project that validates a single file. This is useful for getting language support for files that don't belong to a project + */ + private async createStandaloneProject(srcPath: string) { + const rootDir = path.join(__dirname, 'standalone-project'); + await this.createProject({ + //these folders don't matter for standalone projects + workspaceFolder: rootDir, + projectPath: rootDir, + threadingEnabled: false, + files: [{ + src: srcPath, + dest: 'source/standalone.brs' + }] + }); } /** @@ -149,7 +176,6 @@ export class ProjectManager { await this.handleFileChange(change); })); }); - await this.handleFileChangesPromise; return this.handleFileChangesPromise; } @@ -164,13 +190,8 @@ export class ProjectManager { //file added or changed } else { - //this is a new file. set the file contents - if (fsExtra.statSync(srcPath).isFile()) { - const fileContents = change.fileContents ?? (await fsExtra.readFile(change.srcPath, 'utf8')).toString(); - this.documentManager.set(change.srcPath, fileContents); - - //if this is a new directory, read all files recursively and register those as file changes too - } else { + //if this is a new directory, read all files recursively and register those as file changes too + if (fsExtra.statSync(srcPath).isDirectory()) { const files = await fastGlob('**/*', { cwd: change.srcPath, onlyFiles: true, @@ -180,9 +201,15 @@ export class ProjectManager { await Promise.all(files.map((srcPath) => { return this.handleFileChange({ srcPath: srcPath, - type: FileChangeType.Changed + type: FileChangeType.Changed, + allowStandaloneProject: change.allowStandaloneProject }); })); + + //this is a new file. set the file contents + } else { + const fileContents = change.fileContents ?? (await fsExtra.readFile(change.srcPath, 'utf8')).toString(); + this.documentManager.set(change.srcPath, fileContents, change.allowStandaloneProject); } } @@ -507,7 +534,7 @@ export class ProjectManager { * @returns a new project, or the existing project if one already exists with this config info */ @TrackBusyStatus - private async createProject(config: ProjectConfig) { + private async createProject(config: ProjectConfig): Promise { //skip this project if we already have it if (this.hasProject(config.projectPath)) { return this.getProject(config.projectPath); @@ -529,6 +556,7 @@ export class ProjectManager { config.projectNumber ??= ProjectManager.projectNumberSequence++; await project.activate(config); + return project; } public on(eventName: 'critical-failure', handler: (data: { project: LspProject; message: string }) => MaybePromise); @@ -605,6 +633,15 @@ export interface ProjectConfig { * TODO - is there a better name for this? */ threadingEnabled?: boolean; + /** + * If present, this will override any files array found in bsconfig or the default. + * + * The list of file globs used to find all files for the project + * If using the {src;dest;} format, you can specify a different destination directory + * for the matched files in src. + * + */ + files?: Array; } diff --git a/src/lsp/worker/WorkerThreadProject.ts b/src/lsp/worker/WorkerThreadProject.ts index d0fa79ab0..5f4fd16e2 100644 --- a/src/lsp/worker/WorkerThreadProject.ts +++ b/src/lsp/worker/WorkerThreadProject.ts @@ -9,7 +9,7 @@ import { isMainThread, parentPort } from 'worker_threads'; import { WorkerThreadProjectRunner } from './WorkerThreadProjectRunner'; import { WorkerPool } from './WorkerPool'; import type { Hover, MaybePromise, SemanticToken } from '../../interfaces'; -import type { DocumentAction } from '../DocumentManager'; +import type { DocumentAction, DocumentActionWithStatus } from '../DocumentManager'; import { Deferred } from '../../deferred'; import type { FileTranspileResult, SignatureInfoObj } from '../../Program'; import type { Position, Range, Location, DocumentSymbol, WorkspaceSymbol, CodeAction, CompletionList } from 'vscode-languageserver-protocol'; @@ -28,7 +28,7 @@ export const workerPool = new WorkerPool(() => { ); }); -//if this script us running in a Worker, run +//if this script is running in a Worker, start the project runner /* istanbul ignore next */ if (!isMainThread) { const runner = new WorkerThreadProjectRunner(); @@ -150,18 +150,16 @@ export class WorkerThreadProject implements LspProject { } /** - * Set new contents for a file. This is safe to call any time. Changes will be queued and flushed at the correct times + * Apply a series of file changes to the project. This is safe to call any time. Changes will be queued and flushed at the correct times * during the program's lifecycle flow - * @param documentActions absolute source path of the file */ - public async applyFileChanges(documentActions: DocumentAction[]) { - const response = await this.messageHandler.sendRequest('applyFileChanges', { + public async applyFileChanges(documentActions: DocumentAction[]): Promise { + const response = await this.messageHandler.sendRequest('applyFileChanges', { data: [documentActions] }); return response.data; } - /** * Get the list of all file paths that are currently loaded in the project */ From f011561469d4b01ca1516c278ac365235ae7452a Mon Sep 17 00:00:00 2001 From: Bronley Plumb Date: Tue, 26 Mar 2024 14:19:19 -0400 Subject: [PATCH 047/119] Fix tsc, eslint, and test issues. --- src/DiagnosticCollection.ts | 2 +- src/LanguageServer.spec.ts | 2380 ++++++++++---------- src/LanguageServer.ts | 166 +- src/Program.spec.ts | 2 +- src/interfaces.ts | 2 +- src/lsp/DocumentManager.spec.ts | 12 +- src/lsp/DocumentManager.ts | 4 +- src/lsp/LspProject.ts | 36 +- src/lsp/Project.spec.ts | 32 +- src/lsp/Project.ts | 38 +- src/lsp/ProjectManager.ts | 48 +- src/lsp/ReaderWriterManager.ts | 4 +- src/lsp/worker/WorkerThreadProject.spec.ts | 4 +- src/lsp/worker/WorkerThreadProject.ts | 28 +- 14 files changed, 1296 insertions(+), 1462 deletions(-) diff --git a/src/DiagnosticCollection.ts b/src/DiagnosticCollection.ts index c0ce905cd..b2861b13d 100644 --- a/src/DiagnosticCollection.ts +++ b/src/DiagnosticCollection.ts @@ -1,5 +1,5 @@ import { URI } from 'vscode-uri'; -import type { LspDiagnostic, LspProject } from './lsp/LspProject'; +import type { LspDiagnostic } from './lsp/LspProject'; import { util } from './util'; import { firstBy } from 'thenby'; diff --git a/src/LanguageServer.spec.ts b/src/LanguageServer.spec.ts index 80df85943..c953a87a0 100644 --- a/src/LanguageServer.spec.ts +++ b/src/LanguageServer.spec.ts @@ -1,1190 +1,1190 @@ -import { expect } from './chai-config.spec'; -import * as fsExtra from 'fs-extra'; -import * as path from 'path'; -import type { DidChangeWatchedFilesParams, Location } from 'vscode-languageserver'; -import { FileChangeType, Range } from 'vscode-languageserver'; -import { Deferred } from './deferred'; -import type { Project } from './lsp/Project'; -import { CustomCommands, LanguageServer } from './LanguageServer'; -import type { SinonStub } from 'sinon'; -import { createSandbox } from 'sinon'; -import { standardizePath as s, util } from './util'; -import { TextDocument } from 'vscode-languageserver-textdocument'; -import type { Program } from './Program'; -import * as assert from 'assert'; -import { expectZeroDiagnostics, trim } from './testHelpers.spec'; -import { isBrsFile, isLiteralString } from './astUtils/reflection'; -import { createVisitor, WalkMode } from './astUtils/visitors'; -import { tempDir, rootDir } from './testHelpers.spec'; -import { URI } from 'vscode-uri'; -import { BusyStatusTracker } from './BusyStatusTracker'; -import type { BscFile } from '.'; - -const sinon = createSandbox(); - -const workspacePath = rootDir; - -describe('LanguageServer', () => { - let server: LanguageServer; - let program: Program; - - let workspaceFolders: string[] = []; - - let vfs = {} as Record; - let physicalFilePaths = [] as string[]; - let connection = { - onInitialize: () => null, - onInitialized: () => null, - onDidChangeConfiguration: () => null, - onDidChangeWatchedFiles: () => null, - onCompletion: () => null, - onCompletionResolve: () => null, - onDocumentSymbol: () => null, - onWorkspaceSymbol: () => null, - onDefinition: () => null, - onSignatureHelp: () => null, - onReferences: () => null, - onHover: () => null, - listen: () => null, - sendNotification: () => null, - sendDiagnostics: () => null, - onExecuteCommand: () => null, - onCodeAction: () => null, - onDidOpenTextDocument: () => null, - onDidChangeTextDocument: () => null, - onDidCloseTextDocument: () => null, - onWillSaveTextDocument: () => null, - onWillSaveTextDocumentWaitUntil: () => null, - onDidSaveTextDocument: () => null, - onRequest: () => null, - workspace: { - getWorkspaceFolders: () => { - return workspaceFolders.map( - x => ({ - uri: getFileProtocolPath(x), - name: path.basename(x) - }) - ); - }, - getConfiguration: () => { - return {}; - } - }, - tracer: { - log: () => { } - } - }; - - beforeEach(() => { - sinon.restore(); - server = new LanguageServer(); - server['busyStatusTracker'] = new BusyStatusTracker(); - workspaceFolders = [workspacePath]; - - vfs = {}; - physicalFilePaths = []; - - //hijack the file resolver so we can inject in-memory files for our tests - let originalResolver = server['documentFileResolver']; - server['documentFileResolver'] = (srcPath: string) => { - if (vfs[srcPath]) { - return vfs[srcPath]; - } else { - return originalResolver.call(server, srcPath); - } - }; - - //mock the connection stuff - (server as any).createConnection = () => { - return connection; - }; - server['hasConfigurationCapability'] = true; - }); - afterEach(async () => { - fsExtra.emptyDirSync(tempDir); - try { - await Promise.all( - physicalFilePaths.map(srcPath => fsExtra.unlinkSync(srcPath)) - ); - } catch (e) { - - } - server.dispose(); - }); - - function addXmlFile(name: string, additionalXmlContents = '') { - const filePath = `components/${name}.xml`; - - const contents = ` - - ${additionalXmlContents} -