From 99bfe5fcecd52f2796fa24dfa317b211f4095eef Mon Sep 17 00:00:00 2001 From: mrmeku Date: Sat, 30 Jan 2021 10:15:25 -0700 Subject: [PATCH] feat(typescript): create a better ts_project worker (#2416) --- packages/typescript/BUILD.bazel | 2 +- packages/typescript/internal/ts_project.bzl | 45 ++---- .../typescript/internal/worker/BUILD.bazel | 27 ++-- packages/typescript/internal/worker/index.js | 102 ++++++++++++ .../internal/worker/worker_adapter.js | 99 ------------ .../internal/worker/worker_adapter.ts | 147 ++++++++++++++++++ packages/typescript/replacements.bzl | 5 +- .../test/ts_project/worker/BUILD.bazel | 1 - .../typescript/test/ts_project/worker/big.ts | 2 +- 9 files changed, 284 insertions(+), 146 deletions(-) create mode 100644 packages/typescript/internal/worker/index.js delete mode 100644 packages/typescript/internal/worker/worker_adapter.js create mode 100644 packages/typescript/internal/worker/worker_adapter.ts diff --git a/packages/typescript/BUILD.bazel b/packages/typescript/BUILD.bazel index bb851b493e..64c07e41c7 100644 --- a/packages/typescript/BUILD.bazel +++ b/packages/typescript/BUILD.bazel @@ -96,7 +96,7 @@ pkg_npm( ":npm_version_check", "//packages/typescript/internal:BUILD", "//packages/typescript/internal:ts_project_options_validator.js", - "//packages/typescript/internal/worker", + "//packages/typescript/internal/worker:filegroup", ] + select({ # FIXME: fix stardoc on Windows; //packages/typescript:index.md generation fails with: # ERROR: D:/b/62unjjin/external/npm_bazel_typescript/BUILD.bazel:36:1: Couldn't build file diff --git a/packages/typescript/internal/ts_project.bzl b/packages/typescript/internal/ts_project.bzl index ee49db42da..1eb2e14bc0 100644 --- a/packages/typescript/internal/ts_project.bzl +++ b/packages/typescript/internal/ts_project.bzl @@ -14,14 +14,7 @@ _DEFAULT_TSC = ( "//typescript/bin:tsc" ) -_DEFAULT_TSC_BIN = ( - # BEGIN-INTERNAL - "@npm" + - # END-INTERNAL - "//:node_modules/typescript/bin/tsc" -) - -_DEFAULT_TYPESCRIPT_MODULE = ( +_DEFAULT_TYPESCRIPT_PACKAGE = ( # BEGIN-INTERNAL "@npm" + # END-INTERNAL @@ -49,7 +42,7 @@ _ATTRS = { # that compiler might allow more sources than tsc does. "srcs": attr.label_list(allow_files = True, mandatory = True), "supports_workers": attr.bool(default = False), - "tsc": attr.label(default = Label(_DEFAULT_TSC), executable = True, cfg = "target"), + "tsc": attr.label(default = Label(_DEFAULT_TSC), executable = True, cfg = "host"), "tsconfig": attr.label(mandatory = True, allow_single_file = [".json"]), } @@ -209,6 +202,7 @@ def _ts_project_impl(ctx): inputs = inputs, arguments = [arguments], outputs = outputs, + mnemonic = "TsProject", executable = "tsc", execution_requirements = execution_requirements, progress_message = "%s %s [tsc -p %s]" % ( @@ -337,8 +331,8 @@ def ts_project_macro( emit_declaration_only = False, ts_build_info_file = None, tsc = None, - worker_tsc_bin = _DEFAULT_TSC_BIN, - worker_typescript_module = _DEFAULT_TYPESCRIPT_MODULE, + typescript_package = _DEFAULT_TYPESCRIPT_PACKAGE, + typescript_require_path = "typescript", validate = True, supports_workers = False, declaration_dir = None, @@ -506,14 +500,13 @@ def ts_project_macro( For example, `tsc = "@my_deps//typescript/bin:tsc"` Or you can pass a custom compiler binary instead. - worker_tsc_bin: Label of the TypeScript compiler binary to run when running in worker mode. + typescript_package: Label of the package containing all data deps of tsc. - For example, `tsc = "@my_deps//node_modules/typescript/bin/tsc"` - Or you can pass a custom compiler binary instead. + For example, `typescript_package = "@my_deps//typescript"` - worker_typescript_module: Label of the package containing all data deps of worker_tsc_bin. + typescript_require_path: Module name which resolves to typescript_package when required - For example, `tsc = "@my_deps//typescript"` + For example, `typescript_require_path = "typescript"` validate: boolean; whether to check that the tsconfig settings match the attributes. @@ -642,25 +635,18 @@ def ts_project_macro( # but that's our own code, so we don't. "@npm//protobufjs", # END-INTERNAL - Label("//packages/typescript/internal/worker:worker"), - Label(worker_tsc_bin), - Label(worker_typescript_module), + Label(typescript_package), + Label("//packages/typescript/internal/worker:filegroup"), tsconfig, ], entry_point = Label("//packages/typescript/internal/worker:worker_adapter"), templated_args = [ - "$(execpath {})".format(Label(worker_tsc_bin)), - "--project", - "$(execpath {})".format(tsconfig), - # FIXME: should take out_dir into account - "--outDir", - "$(RULEDIR)", - # FIXME: what about other settings like declaration_dir, root_dir, etc + "--typescript_require_path", + typescript_require_path, ], ) tsc = ":" + tsc_worker - typings_out_dir = declaration_dir if declaration_dir else out_dir tsbuildinfo_path = ts_build_info_file if ts_build_info_file else name + ".tsbuildinfo" js_outs = [] @@ -701,9 +687,6 @@ Check the srcs attribute to see that some .ts files are present (or .js files wi buildinfo_out = tsbuildinfo_path if composite or incremental else None, tsc = tsc, link_workspace_root = link_workspace_root, - supports_workers = select({ - "@bazel_tools//src/conditions:host_windows": False, - "//conditions:default": supports_workers, - }), + supports_workers = supports_workers, **kwargs ) diff --git a/packages/typescript/internal/worker/BUILD.bazel b/packages/typescript/internal/worker/BUILD.bazel index f6593042a2..ece8edbef1 100644 --- a/packages/typescript/internal/worker/BUILD.bazel +++ b/packages/typescript/internal/worker/BUILD.bazel @@ -1,8 +1,18 @@ # BEGIN-INTERNAL - -load("//internal/common:copy_to_bin.bzl", "copy_to_bin") +load("//packages/typescript:checked_in_ts_project.bzl", "checked_in_ts_project") load("//third_party/github.com/bazelbuild/bazel-skylib:rules/copy_file.bzl", "copy_file") +# To update index.js run: +# bazel run //packages/typescript/internal/worker:worker_adapter_check_compiled.update + +checked_in_ts_project( + name = "worker_adapter", + src = "worker_adapter.ts", + checked_in_js = "index.js", + visibility = ["//visibility:public"], + deps = ["@npm//@types/node"], +) + # Copy the proto file to a matching third_party/... nested directory # so the runtime require() statements still work _worker_proto_dir = "third_party/github.com/bazelbuild/bazel/src/main/protobuf" @@ -22,14 +32,6 @@ copy_file( visibility = ["//visibility:public"], ) -copy_to_bin( - name = "worker_adapter", - srcs = [ - "worker_adapter.js", - ], - visibility = ["//visibility:public"], -) - filegroup( name = "package_contents", srcs = [ @@ -41,12 +43,13 @@ filegroup( # END-INTERNAL exports_files([ - "worker_adapter.js", + "index.js", ]) filegroup( - name = "worker", + name = "filegroup", srcs = [ + "index.js", "third_party/github.com/bazelbuild/bazel/src/main/protobuf/worker_protocol.proto", "worker.js", "worker_adapter.js", diff --git a/packages/typescript/internal/worker/index.js b/packages/typescript/internal/worker/index.js new file mode 100644 index 0000000000..fc12cbdea3 --- /dev/null +++ b/packages/typescript/internal/worker/index.js @@ -0,0 +1,102 @@ +/* THIS FILE GENERATED FROM .ts; see BUILD.bazel */ /* clang-format off */"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const fs = require("fs"); +const ts = require("typescript"); +const MNEMONIC = 'TsProject'; +const worker = require('./worker'); +let createWatchCompilerHost; +const formatHost = { + getCanonicalFileName: (path) => path, + getCurrentDirectory: ts.sys.getCurrentDirectory, + getNewLine: () => ts.sys.newLine, +}; +const reportDiagnostic = (diagnostic) => { + worker.log(ts.formatDiagnostic(diagnostic, formatHost)); +}; +const reportWatchStatusChanged = (diagnostic) => { + worker.debug(ts.formatDiagnostic(diagnostic, formatHost)); +}; +function createWatchProgram(options, tsconfigPath, setTimeout) { + const host = createWatchCompilerHost(tsconfigPath, options, Object.assign(Object.assign({}, ts.sys), { setTimeout }), ts.createEmitAndSemanticDiagnosticsBuilderProgram, reportDiagnostic, reportWatchStatusChanged); + return ts.createWatchProgram(host); +} +let workerRequestTimestamp; +let cachedWatchedProgram; +let consolidateChangesCallback; +let cachedWatchProgramArgs; +function getWatchProgram(args) { + const newWatchArgs = args.join(' '); + if (cachedWatchedProgram && cachedWatchProgramArgs && cachedWatchProgramArgs !== newWatchArgs) { + cachedWatchedProgram.close(); + cachedWatchedProgram = undefined; + cachedWatchProgramArgs = undefined; + } + if (!cachedWatchedProgram) { + const parsedArgs = ts.parseCommandLine(args); + const tsconfigPath = args[args.indexOf('--project') + 1]; + cachedWatchProgramArgs = newWatchArgs; + cachedWatchedProgram = createWatchProgram(parsedArgs.options, tsconfigPath, (callback) => { + consolidateChangesCallback = callback; + }); + } + return cachedWatchedProgram; +} +function emitOnce(args) { + var _a; + return __awaiter(this, void 0, void 0, function* () { + const watchProgram = getWatchProgram(args); + if (consolidateChangesCallback) { + consolidateChangesCallback(); + } + workerRequestTimestamp = Date.now(); + const result = yield ((_a = watchProgram) === null || _a === void 0 ? void 0 : _a.getProgram().emit(undefined, undefined, { + isCancellationRequested: function (timestamp) { + return timestamp !== workerRequestTimestamp; + }.bind(null, workerRequestTimestamp), + throwIfCancellationRequested: function (timestamp) { + if (timestamp !== workerRequestTimestamp) { + throw new ts.OperationCanceledException(); + } + }.bind(null, workerRequestTimestamp), + })); + return Boolean(result && result.diagnostics.length === 0); + }); +} +function main() { + const typescriptRequirePath = process.argv[process.argv.indexOf('--typescript_require_path') + 1]; + try { + const customTypescriptModule = require(typescriptRequirePath); + createWatchCompilerHost = customTypescriptModule.createWatchCompilerHost; + } + catch (e) { + worker.log(`typescript_require_path '${typescriptRequirePath}' could not be resolved`); + throw e; + } + if (process.argv.includes('--persistent_worker')) { + worker.log(`Running ${MNEMONIC} as a Bazel worker`); + worker.runWorkerLoop(emitOnce); + } + else { + worker.log(`Running ${MNEMONIC} as a standalone process`); + worker.log(`Started a new process to perform this action. Your build might be misconfigured, try + --strategy=${MNEMONIC}=worker`); + let argsFilePath = process.argv.pop(); + if (argsFilePath.startsWith('@')) { + argsFilePath = argsFilePath.slice(1); + } + const args = fs.readFileSync(argsFilePath).toString().split('\n'); + emitOnce(args).finally(() => { var _a; return (_a = cachedWatchedProgram) === null || _a === void 0 ? void 0 : _a.close(); }); + } +} +if (require.main === module) { + main(); +} diff --git a/packages/typescript/internal/worker/worker_adapter.js b/packages/typescript/internal/worker/worker_adapter.js deleted file mode 100644 index 38240228b7..0000000000 --- a/packages/typescript/internal/worker/worker_adapter.js +++ /dev/null @@ -1,99 +0,0 @@ -/** - * @fileoverview wrapper program around the TypeScript compiler, tsc - * - * It intercepts the Bazel Persistent Worker protocol, using it to remote-control tsc running as a - * child process. In between builds, the tsc process is stopped (akin to ctrl-z in a shell) and then - * resumed (akin to `fg`) when the inputs have changed. - * - * See https://medium.com/@mmorearty/how-to-create-a-persistent-worker-for-bazel-7738bba2cabb - * for more background (note, that is documenting a different implementation) - */ -const child_process = require('child_process'); -const MNEMONIC = 'TsProject'; -const worker = require('./worker'); - -const workerArg = process.argv.indexOf('--persistent_worker') -if (workerArg > 0) { - process.argv.splice(workerArg, 1, '--watch') - - if (process.platform !== 'linux' && process.platform !== 'darwin') { - throw new Error(`Worker mode is only supported on linux and darwin, not ${process.platform}. - See https://github.com/bazelbuild/rules_nodejs/issues/2277`); - } -} - -const [tscBin, ...tscArgs] = process.argv.slice(2); - -const child = child_process.spawn( - tscBin, - tscArgs, - {stdio: 'pipe'}, -); -function awaitOneBuild() { - child.kill('SIGCONT') - - let buffer = []; - return new Promise((res) => { - function awaitBuild(s) { - buffer.push(s); - - if (s.includes('Watching for file changes.')) { - child.kill('SIGSTOP') - - const success = s.includes('Found 0 errors.'); - res(success); - - child.stdout.removeListener('data', awaitBuild); - - if (!success) { - console.error( - `\nError output from tsc worker:\n\n ${ - buffer.slice(1).map(s => s.toString()).join('').replace(/\n/g, '\n ')}`, - ) - } - - buffer = []; - } - }; - child.stdout.on('data', awaitBuild); - }); -} - -async function main() { - // Bazel will pass a special argument to the program when it's running us as a worker - if (workerArg > 0) { - worker.log(`Running ${MNEMONIC} as a Bazel worker`); - - worker.runWorkerLoop(awaitOneBuild); - } else { - // Running standalone so stdout is available as usual - console.log(`Running ${MNEMONIC} as a standalone process`); - console.error( - `Started a new process to perform this action. Your build might be misconfigured, try - --strategy=${MNEMONIC}=worker`); - - const stdoutbuffer = []; - child.stdout.on('data', data => stdoutbuffer.push(data)); - - const stderrbuffer = []; - child.stderr.on('data', data => stderrbuffer.push(data)); - - child.on('exit', code => { - if (code !== 0) { - console.error( - `\nstdout from tsc:\n\n ${ - stdoutbuffer.map(s => s.toString()).join('').replace(/\n/g, '\n ')}`, - ) - console.error( - `\nstderr from tsc:\n\n ${ - stderrbuffer.map(s => s.toString()).join('').replace(/\n/g, '\n ')}`, - ) - } - process.exit(code) - }); - } -} - -if (require.main === module) { - main(); -} diff --git a/packages/typescript/internal/worker/worker_adapter.ts b/packages/typescript/internal/worker/worker_adapter.ts new file mode 100644 index 0000000000..3f6f7c218e --- /dev/null +++ b/packages/typescript/internal/worker/worker_adapter.ts @@ -0,0 +1,147 @@ +/** + * @fileoverview wrapper program around the TypeScript Watcher Compiler Host. + * + * It intercepts the Bazel Persistent Worker protocol, using it to + * remote-control compiler host. It tells the compiler process to + * consolidate file changes only when it receives a request from the worker + * protocol. + * + * See https://medium.com/@mmorearty/how-to-create-a-persistent-worker-for-bazel-7738bba2cabb + * for more background on the worker protocol. + */ +import * as fs from 'fs'; +import * as ts from 'typescript'; + +const MNEMONIC = 'TsProject'; +const worker = require('./worker'); + +let createWatchCompilerHost: typeof ts.createWatchCompilerHost; + +const formatHost: ts.FormatDiagnosticsHost = { + getCanonicalFileName: (path) => path, + getCurrentDirectory: ts.sys.getCurrentDirectory, + getNewLine: () => ts.sys.newLine, +}; + +/** + * Prints a diagnostic result for every compiler error or warning. + */ +const reportDiagnostic: ts.DiagnosticReporter = (diagnostic) => { + worker.log(ts.formatDiagnostic(diagnostic, formatHost)); +}; + +/** + * Prints a diagnostic every time the watch status changes. + * This is mainly for messages like "Starting compilation" or "Compilation completed". + */ +const reportWatchStatusChanged: ts.WatchStatusReporter = (diagnostic) => { + worker.debug(ts.formatDiagnostic(diagnostic, formatHost)); +}; + +function createWatchProgram( + options: ts.CompilerOptions, tsconfigPath: string, setTimeout: ts.System['setTimeout']) { + const host = createWatchCompilerHost( + tsconfigPath, options, {...ts.sys, setTimeout}, + ts.createEmitAndSemanticDiagnosticsBuilderProgram, reportDiagnostic, + reportWatchStatusChanged); + + // `createWatchProgram` creates an initial program, watches files, and updates + // the program over time. + return ts.createWatchProgram(host); +} + +/** + * Timestamp of the last worker request. + */ +let workerRequestTimestamp: number|undefined; +/** + * The typescript compiler in watch mode. + */ +let cachedWatchedProgram:|ts.WatchOfConfigFile| + undefined; +/** + * Callback provided by ts.System which should be called at the point at which + * file system changes should be consolidated into a new emission from the + * watcher. + */ +let consolidateChangesCallback: ((...args: any[]) => void)|undefined; +let cachedWatchProgramArgs: string|undefined; + +function getWatchProgram(args: string[]): + ts.WatchOfConfigFile { + const newWatchArgs = args.join(' '); + + // Check to see if the watch program needs to be updated or if we can re-use the old one. + if (cachedWatchedProgram && cachedWatchProgramArgs && cachedWatchProgramArgs !== newWatchArgs) { + cachedWatchedProgram.close(); + cachedWatchedProgram = undefined; + cachedWatchProgramArgs = undefined; + } + + // If we have not yet created a watch + if (!cachedWatchedProgram) { + const parsedArgs = ts.parseCommandLine(args); + const tsconfigPath = args[args.indexOf('--project') + 1]; + + cachedWatchProgramArgs = newWatchArgs; + cachedWatchedProgram = createWatchProgram(parsedArgs.options, tsconfigPath, (callback) => { + consolidateChangesCallback = callback; + }); + } + + return cachedWatchedProgram; +} + +async function emitOnce(args: string[]) { + const watchProgram = getWatchProgram(args); + + if (consolidateChangesCallback) { + consolidateChangesCallback(); + } + + + workerRequestTimestamp = Date.now(); + const result = await watchProgram ?.getProgram().emit(undefined, undefined, { + isCancellationRequested: function(timestamp: number) { + return timestamp !== workerRequestTimestamp; + }.bind(null, workerRequestTimestamp), + throwIfCancellationRequested: function(timestamp: number) { + if (timestamp !== workerRequestTimestamp) { + throw new ts.OperationCanceledException(); + } + }.bind(null, workerRequestTimestamp), + }); + + return Boolean(result && result.diagnostics.length === 0); +} + +function main() { + const typescriptRequirePath = process.argv[process.argv.indexOf('--typescript_require_path') + 1]; + try { + const customTypescriptModule = require(typescriptRequirePath); + createWatchCompilerHost = customTypescriptModule.createWatchCompilerHost; + } catch (e) { + worker.log(`typescript_require_path '${typescriptRequirePath}' could not be resolved`) + throw e; + } + if (process.argv.includes('--persistent_worker')) { + worker.log(`Running ${MNEMONIC} as a Bazel worker`); + worker.runWorkerLoop(emitOnce); + } else { + worker.log(`Running ${MNEMONIC} as a standalone process`); + worker.log( + `Started a new process to perform this action. Your build might be misconfigured, try + --strategy=${MNEMONIC}=worker`); + + let argsFilePath = process.argv.pop()!; + if (argsFilePath.startsWith('@')) { + argsFilePath = argsFilePath.slice(1) + } + const args = fs.readFileSync(argsFilePath).toString().split('\n'); + emitOnce(args).finally(() => cachedWatchedProgram?.close()); + } +} + +if (require.main === module) { + main(); +} diff --git a/packages/typescript/replacements.bzl b/packages/typescript/replacements.bzl index 5665ffe7a7..aa4d1899d2 100644 --- a/packages/typescript/replacements.bzl +++ b/packages/typescript/replacements.bzl @@ -24,7 +24,10 @@ TYPESCRIPT_REPLACEMENTS = dict( # @build_bazel_rules_typescript//:npm_bazel_typescript_package # use this alternate fencing "(#|\\/\\/)\\s+BEGIN-DEV-ONLY[\\w\\W]+?(#|\\/\\/)\\s+END-DEV-ONLY": "", - "//packages/typescript/internal/worker:worker_adapter": "//@bazel/typescript/internal/worker:worker_adapter.js", + # Replace the worker filegroup with the entire @bazel/typescript node_module and its transitive node_modules + "//packages/typescript/internal/worker:filegroup": "//@bazel/typescript", + # Change the worker entry point from the checked_in_ts_project target to the checked in .js + "//packages/typescript/internal/worker:worker_adapter": "//@bazel/typescript/internal/worker:index.js", # This file gets vendored into our repo "@build_bazel_rules_typescript//internal:common": "//@bazel/typescript/internal:common", # Replace the local compiler label with one that comes from npm diff --git a/packages/typescript/test/ts_project/worker/BUILD.bazel b/packages/typescript/test/ts_project/worker/BUILD.bazel index 73924e9036..1abe48d0de 100644 --- a/packages/typescript/test/ts_project/worker/BUILD.bazel +++ b/packages/typescript/test/ts_project/worker/BUILD.bazel @@ -2,7 +2,6 @@ load("//packages/typescript:index.bzl", "ts_project") ts_project( supports_workers = True, - tags = ["fix-windows"], tsconfig = { "compilerOptions": { "declaration": True, diff --git a/packages/typescript/test/ts_project/worker/big.ts b/packages/typescript/test/ts_project/worker/big.ts index 124564061e..6d90a44168 100644 --- a/packages/typescript/test/ts_project/worker/big.ts +++ b/packages/typescript/test/ts_project/worker/big.ts @@ -1,3 +1,3 @@ // TODO: make it big enough to slow down tsc -export const a: number = 2; +export const a: number = 2350;