From b187971dc77b8d05935151d8d3a8a3e8c239e6d7 Mon Sep 17 00:00:00 2001 From: brightwu <1521488775@qq.com> Date: Sat, 11 Mar 2023 18:46:06 +0800 Subject: [PATCH] chore: optimize cold start speed and fix lazy compilation issue (#70) * chore: add profiler and debug compilation slow issue on windows * chore: optimize cold start speed * chore: update lockfile --- Cargo.lock | 315 ++++++++++++++++++ crates/compiler/src/lib.rs | 2 + crates/node/Cargo.toml | 12 + crates/node/src/lib.rs | 49 ++- crates/plugin_resolve/src/lib.rs | 16 +- crates/plugin_resolve/src/resolver.rs | 13 +- cspell.json | 3 +- packages/core/CHANGELOG.md | 9 + packages/core/binding/binding.d.ts | 1 + packages/core/package.json | 6 +- packages/core/src/compiler/index.ts | 17 + packages/core/src/index.ts | 20 +- packages/core/src/logger.ts | 2 +- packages/core/src/server/hmr-engine.ts | 41 ++- packages/core/src/server/index.ts | 8 +- packages/core/src/server/middlewares/hmr.ts | 2 +- .../server/middlewares/lazy-compilation.ts | 19 +- packages/core/src/watcher/index.ts | 44 ++- packages/runtime-plugin-hmr/CHANGELOG.md | 6 + packages/runtime-plugin-hmr/package.json | 4 +- packages/runtime-plugin-hmr/src/index.ts | 5 +- packages/runtime/CHANGELOG.md | 6 + packages/runtime/package.json | 2 +- packages/runtime/src/module-system.ts | 8 +- pnpm-lock.yaml | 6 +- 25 files changed, 553 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bba9a729b..6425fe261 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,6 +73,17 @@ dependencies = [ "syn", ] +[[package]] +name = "async-trait" +version = "0.1.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -168,6 +179,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + [[package]] name = "bytecheck" version = "0.6.9" @@ -189,6 +206,12 @@ dependencies = [ "syn", ] +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "cc" version = "1.0.79" @@ -442,7 +465,10 @@ dependencies = [ "napi", "napi-build", "napi-derive", + "opentelemetry", + "opentelemetry-jaeger", "regex", + "tracing-opentelemetry", ] [[package]] @@ -601,6 +627,95 @@ dependencies = [ "syn", ] +[[package]] +name = "futures" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" + +[[package]] +name = "futures-executor" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" + +[[package]] +name = "futures-macro" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" + +[[package]] +name = "futures-task" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" + +[[package]] +name = "futures-util" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -702,6 +817,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "inventory" version = "0.1.11" @@ -749,6 +870,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1072,6 +1202,86 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +[[package]] +name = "opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e785d273968748578931e4dc3b4f5ec86b26e09d9e0d66b55adda7fce742f7a" +dependencies = [ + "async-trait", + "futures", + "futures-executor", + "once_cell", + "opentelemetry", + "opentelemetry-semantic-conventions", + "thiserror", + "thrift", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b02e0230abb0ab6636d18e2ba8fa02903ea63772281340ccac18e0af3ec9eeb" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_api" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand", + "thiserror", +] + +[[package]] +name = "ordered-float" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" +dependencies = [ + "num-traits", +] + [[package]] name = "overload" version = "0.1.1" @@ -1188,6 +1398,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pmutil" version = "0.5.3" @@ -1590,6 +1806,15 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.9.0" @@ -2443,6 +2668,28 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09678c4cdbb4eed72e18b7c2af1329c69825ed16fcbac62d083fc3e2b0590ff0" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float", + "threadpool", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2514,6 +2761,20 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-subscriber" version = "0.3.16" @@ -2627,6 +2888,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + [[package]] name = "winapi" version = "0.3.9" diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 61fa5d049..1e7b01204 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use farmfe_core::{ config::Config, context::CompilationContext, error::Result, plugin::Plugin, stats::Stats, }; +use farmfe_toolkit::tracing; use update::{UpdateResult, UpdateType}; pub mod build; @@ -51,6 +52,7 @@ impl Compiler { } /// Compile the project using the configuration + #[tracing::instrument(skip_all)] pub fn compile(&self) -> Result<()> { // triggering build stage self.build()?; diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 33bff3787..6a106ec4e 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -19,6 +19,18 @@ farmfe_core = { path = "../core" } farmfe_toolkit = { path = "../toolkit" } regex = "1" libloading = "0.7" +opentelemetry = { version = "0.18.0", default-features = false, features = [ + "trace", +], optional = true } +tracing-opentelemetry = { version = "0.18.0", optional = true } +opentelemetry-jaeger = { version = "0.17.0", optional = true } + +[features] +profiler = [ + "dep:opentelemetry", + "dep:tracing-opentelemetry", + "dep:opentelemetry-jaeger", +] [build-dependencies] napi-build = "2.0.1" diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index 765fadbd7..47641819a 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -89,16 +89,32 @@ impl JsCompiler { plugins_adapters.push(rust_plugin); } - let fmt_layer = fmt::layer().with_target(false); - let filter_layer = EnvFilter::try_from_default_env() - .or_else(|_| EnvFilter::try_new("info")) - .unwrap(); + #[cfg(not(feature = "profile"))] + { + let fmt_layer = fmt::layer().with_target(false); + let filter_layer = EnvFilter::try_from_default_env() + .or_else(|_| EnvFilter::try_new("info")) + .unwrap(); + + tracing_subscriber::registry() + .with(filter_layer) + .with(fmt_layer) + .try_init() + .err(); + } - tracing_subscriber::registry() - .with(filter_layer) - .with(fmt_layer) - .try_init() - .err(); + #[cfg(feature = "profile")] + { + let tracer = opentelemetry_jaeger::new_agent_pipeline() + .with_service_name("farm_profile_pnpm") + .install_simple() + .unwrap(); + let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); + tracing_subscriber::registry() + .with(opentelemetry) + .try_init() + .err(); + } Ok(Self { compiler: Compiler::new(config, plugins_adapters) @@ -116,6 +132,9 @@ impl JsCompiler { .compile() .map_err(|e| napi::Error::new(Status::GenericFailure, format!("{}", e)))?; + #[cfg(feature = "profile")] + opentelemetry::global::shutdown_tracer_provider(); + Ok(()) } @@ -207,6 +226,18 @@ impl JsCompiler { result } + #[napi] + pub fn relative_module_paths(&self) -> Vec { + let context = self.compiler.context(); + let module_graph = context.module_graph.read(); + + module_graph + .modules() + .into_iter() + .map(|m| m.id.relative_path().to_string()) + .collect() + } + #[napi] pub fn resource(&self, name: String) -> Option { let context = self.compiler.context(); diff --git a/crates/plugin_resolve/src/lib.rs b/crates/plugin_resolve/src/lib.rs index 4df177f2d..34f500a6c 100644 --- a/crates/plugin_resolve/src/lib.rs +++ b/crates/plugin_resolve/src/lib.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, path::Path, sync::Arc}; +use std::{path::Path, sync::Arc}; use farmfe_core::{ config::Config, @@ -6,6 +6,7 @@ use farmfe_core::{ error::Result, plugin::{Plugin, PluginHookContext, PluginResolveHookParam, PluginResolveHookResult}, }; +use farmfe_toolkit::tracing; use farmfe_utils::parse_query; use resolver::Resolver; @@ -29,6 +30,7 @@ impl Plugin for FarmPluginResolve { "FarmPluginResolve" } + #[tracing::instrument(skip_all)] fn resolve( &self, param: &PluginResolveHookParam, @@ -40,7 +42,7 @@ impl Plugin for FarmPluginResolve { // split query from source let splits: Vec<&str> = source.split('?').collect(); let source = splits[0]; - + let basedir = if let Some(importer) = ¶m.importer { if let Some(p) = Path::new(&importer.resolved_path(&context.config.root)).parent() { p.to_path_buf() @@ -62,10 +64,10 @@ impl Plugin for FarmPluginResolve { } let resolver = Resolver::new(context.config.resolve.clone()); - Ok(resolver.resolve(source, basedir, ¶m.kind).map(|result| { - PluginResolveHookResult { - query, - ..result - }})) + Ok( + resolver + .resolve(source, basedir.clone(), ¶m.kind) + .map(|result| PluginResolveHookResult { query, ..result }), + ) } } diff --git a/crates/plugin_resolve/src/resolver.rs b/crates/plugin_resolve/src/resolver.rs index f16b1cf63..3a0319e4d 100644 --- a/crates/plugin_resolve/src/resolver.rs +++ b/crates/plugin_resolve/src/resolver.rs @@ -1,5 +1,4 @@ use std::{ - collections::HashMap, path::{Path, PathBuf}, str::FromStr, }; @@ -12,7 +11,10 @@ use farmfe_core::{ relative_path::RelativePath, serde_json::{from_str, Map, Value}, }; -use farmfe_toolkit::resolve::{follow_symlinks, load_package_json, package_json_loader::Options}; +use farmfe_toolkit::{ + resolve::{follow_symlinks, load_package_json, package_json_loader::Options}, + tracing, +}; pub struct Resolver { config: ResolveConfig, @@ -33,6 +35,7 @@ impl Resolver { /// * **exports**: refer to [exports](https://nodejs.org/api/packages.html#packages_conditional_exports), if source is end with '.js', also try to find '.ts' file /// * **browser**: refer to [package-browser-field-spec](https://github.com/defunctzombie/package-browser-field-spec) /// * **module/main**: `{ "module": "es/index.mjs", "main": "lib/index.cjs" }` + #[tracing::instrument(skip_all)] pub fn resolve( &self, source: &str, @@ -116,7 +119,8 @@ impl Resolver { } } - /// Try resolve as a file with the configured main fields. + /// Try resolve as a file with the configured main fields. + #[tracing::instrument(skip_all)] fn try_directory(&self, dir: &PathBuf) -> Option { if !dir.is_dir() { return None; @@ -135,6 +139,7 @@ impl Resolver { /// Try resolve as a file with the configured extensions. /// If `/root/index` exists, return `/root/index`, otherwise try `/root/index.[configured extension]` in order, once any extension exists (like `/root/index.ts`), return it immediately + #[tracing::instrument(skip_all)] fn try_file(&self, file: &PathBuf) -> Option { // TODO add a test that for directory imports like `import 'comps/button'` where comps/button is a dir if file.exists() && file.is_file() { @@ -157,6 +162,7 @@ impl Resolver { } } + #[tracing::instrument(skip_all)] fn try_alias( &self, source: &str, @@ -180,6 +186,7 @@ impl Resolver { } /// Resolve the source as a package + #[tracing::instrument(skip_all)] fn try_node_modules( &self, source: &str, diff --git a/cspell.json b/cspell.json index 73444615d..9be355720 100644 --- a/cspell.json +++ b/cspell.json @@ -52,7 +52,8 @@ "loglevel", "Pluggable", "Rustup", - "clsx" + "clsx", + "opentelemetry" ], "ignorePaths": [ "pnpm-lock.yaml", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index ff6f84642..efd794026 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,14 @@ # @farmfe/core +## 0.4.2 + +### Patch Changes + +- limit the watched files to optimize cold start speed and fix lazy compilation issue" +- Updated dependencies + - @farmfe/runtime-plugin-hmr@3.0.5 + - @farmfe/runtime@0.3.3 + ## 0.4.1 ### Patch Changes diff --git a/packages/core/binding/binding.d.ts b/packages/core/binding/binding.d.ts index ad6a8cdc4..45af1f62f 100644 --- a/packages/core/binding/binding.d.ts +++ b/packages/core/binding/binding.d.ts @@ -43,5 +43,6 @@ export class Compiler { updateSync(paths: Array): JsUpdateResult; hasModule(resolvedPath: string): boolean; resources(): Record; + relativeModulePaths(): Array; resource(name: string): Buffer | null; } diff --git a/packages/core/package.json b/packages/core/package.json index ce14e8108..a429c3053 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@farmfe/core", - "version": "0.4.1", + "version": "0.4.2", "main": "dist/index.js", "types": "dist/index.d.ts", "type": "module", @@ -62,8 +62,8 @@ "type-check": "tsc -p tsconfig.build.json --noEmit" }, "dependencies": { - "@farmfe/runtime": "workspace:^0.3.2", - "@farmfe/runtime-plugin-hmr": "workspace:^3.0.4", + "@farmfe/runtime": "workspace:^0.3.3", + "@farmfe/runtime-plugin-hmr": "workspace:^3.0.5", "@swc/helpers": "^0.4.9", "boxen": "^7.0.1", "chalk": "^5.2.0", diff --git a/packages/core/src/compiler/index.ts b/packages/core/src/compiler/index.ts index 09ec5633c..b798d0288 100644 --- a/packages/core/src/compiler/index.ts +++ b/packages/core/src/compiler/index.ts @@ -3,6 +3,9 @@ import path from 'node:path'; import type { Config, JsUpdateResult } from '../../binding/index.js'; import { Compiler as BindingCompiler } from '../../binding/index.js'; +export const VIRTUAL_FARM_DYNAMIC_IMPORT_PREFIX = + 'virtual:FARMFE_DYNAMIC_IMPORT:'; + export class Compiler { private _bindingCompiler: BindingCompiler; @@ -85,4 +88,18 @@ export class Compiler { rmSync(outputPath, { recursive: true }); } } + + resolvedModulePaths(root: string): string[] { + return this._bindingCompiler + .relativeModulePaths() + .map((p) => this.transformModulePath(root, p)); + } + + transformModulePath(root: string, p: string): string { + if (p.startsWith(VIRTUAL_FARM_DYNAMIC_IMPORT_PREFIX)) { + return p.slice(VIRTUAL_FARM_DYNAMIC_IMPORT_PREFIX.length); + } + + return path.join(root, p); + } } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 72ca90bd8..af86113f4 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -29,16 +29,24 @@ export async function start(options: { const compiler = new Compiler(normalizedConfig); const devServer = new DevServer(compiler, logger, userConfig.server); + await devServer.listen(); + // Make sure the server is listening before we watch for file changes if (devServer.config.hmr) { logger.info( 'HMR enabled, watching for file changes under ' + chalk.green(userConfig.root) ); + + if (normalizedConfig.config.mode === 'production') { + logger.error( + 'HMR can not be enabled in production mode. Please set the mode option to "development" in your config file.' + ); + process.exit(1); + } + const fileWatcher = new FileWatcher(userConfig.root, devServer.config.hmr); fileWatcher.watch(devServer); } - - devServer.listen(); } export async function build(options: { @@ -60,5 +68,11 @@ export async function build(options: { compiler.removeOutputPathDir(); await compiler.compile(); compiler.writeResourcesToDisk(); - logger.info(`Build completed in ${chalk.green(`${Date.now() - start}ms`)}!`); + logger.info( + `Build completed in ${chalk.green( + `${Date.now() - start}ms` + )}! Resources emitted to ${chalk.green( + normalizedConfig.config.output.path + )}.` + ); } diff --git a/packages/core/src/logger.ts b/packages/core/src/logger.ts index 1e62d0af7..d4b047365 100644 --- a/packages/core/src/logger.ts +++ b/packages/core/src/logger.ts @@ -29,6 +29,6 @@ export class DefaultLogger implements Logger { } error(message: string): void { - log.error(message); + log.error(chalk.red(message)); } } diff --git a/packages/core/src/server/hmr-engine.ts b/packages/core/src/server/hmr-engine.ts index 72b51e190..4efe715da 100644 --- a/packages/core/src/server/hmr-engine.ts +++ b/packages/core/src/server/hmr-engine.ts @@ -1,19 +1,25 @@ // queue all updates and compile them one by one -import { Compiler } from '../compiler/index.js'; +import { + Compiler, + VIRTUAL_FARM_DYNAMIC_IMPORT_PREFIX, +} from '../compiler/index.js'; import { DevServer } from './index.js'; // import debounce from 'lodash.debounce'; import { Logger } from '../logger.js'; import { relative } from 'path'; import chalk from 'chalk'; import type { Resource } from '@farmfe/runtime/src/resource-loader.js'; +import { JsUpdateResult } from '../../binding/binding.js'; export class HmrEngine { private _updateQueue: string[] = []; - private _updateResults: Map = new Map(); + private _updateResults: Map = + new Map(); private _compiler: Compiler; private _devServer: DevServer; + private _onUpdates: ((result: JsUpdateResult) => void)[]; constructor( compiler: Compiler, @@ -24,7 +30,14 @@ export class HmrEngine { this._devServer = devServer; } - recompileAndSendResult = async (): Promise => { + onUpdate(cb: (result: JsUpdateResult) => void) { + if (!this._onUpdates) { + this._onUpdates = []; + } + this._onUpdates.push(cb); + } + + recompileAndSendResult = async (): Promise => { const queue = [...this._updateQueue]; if (queue.length === 0) { @@ -70,10 +83,16 @@ export class HmrEngine { dynamicResourcesMap: ${JSON.stringify(dynamicResourcesMap)} }`; + this._onUpdates?.forEach((cb) => cb(result)); + const id = Date.now().toString(); // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore TODO fix this - this._updateResults.set(id, resultStr); + this._updateResults.set(id, { + result: resultStr, + count: this._devServer.ws.clients.size, + }); + // console.log(this._updateResults); this._devServer.ws.clients.forEach((client) => { client.send( @@ -92,7 +111,7 @@ export class HmrEngine { async hmrUpdate(path: string) { // if lazy compilation is enabled, we need to update the virtual module if (this._compiler.config.config.lazyCompilation) { - const lazyCompiledModule = `virtual:FARMFE_DYNAMIC_IMPORT:${path}`; + const lazyCompiledModule = `${VIRTUAL_FARM_DYNAMIC_IMPORT_PREFIX}${path}`; if ( this._compiler.hasModule(lazyCompiledModule) && @@ -121,7 +140,15 @@ export class HmrEngine { getUpdateResult(id: string) { const result = this._updateResults.get(id); - this._updateResults.delete(id); - return result; + + if (result) { + result.count--; + // there are no more clients waiting for this update + if (result.count <= 0) { + this._updateResults.delete(id); + } + } + + return result?.result; } } diff --git a/packages/core/src/server/index.ts b/packages/core/src/server/index.ts index 78dc7979f..87e85dd2d 100644 --- a/packages/core/src/server/index.ts +++ b/packages/core/src/server/index.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, readFileSync } from 'fs'; +import { readFileSync } from 'fs'; import Koa from 'koa'; // import serve from 'koa-static'; @@ -30,7 +30,6 @@ import { resources } from './middlewares/resources.js'; */ export class DevServer { private _app: Koa; - private _dist: string; ws: WebSocketServer; config: NormalizedServerConfig; @@ -43,11 +42,6 @@ export class DevServer { ) { this.config = normalizeDevServerOptions(options); this._app = new Koa(); - this._dist = this._compiler.config.config.output.path as string; - - if (!existsSync(this._dist)) { - mkdirSync(this._dist, { recursive: true }); - } // this._app.use(serve(this._dist)); this._app.use(resources(this._compiler)); diff --git a/packages/core/src/server/middlewares/hmr.ts b/packages/core/src/server/middlewares/hmr.ts index bb165ef5b..3823a8cb0 100644 --- a/packages/core/src/server/middlewares/hmr.ts +++ b/packages/core/src/server/middlewares/hmr.ts @@ -9,7 +9,7 @@ import { Context } from 'koa'; import { DevServer } from '../index.js'; export function hmr(server: DevServer) { - return async (ctx: Context, next: () => Promise) => { + return async (ctx: Context, next: () => Promise) => { await next(); if (ctx.path === '/__hmr') { diff --git a/packages/core/src/server/middlewares/lazy-compilation.ts b/packages/core/src/server/middlewares/lazy-compilation.ts index eb3881a31..890657353 100644 --- a/packages/core/src/server/middlewares/lazy-compilation.ts +++ b/packages/core/src/server/middlewares/lazy-compilation.ts @@ -7,6 +7,7 @@ import chalk from 'chalk'; import { DevServer } from '../index.js'; import type { Resource } from '@farmfe/runtime/src/resource-loader.js'; +import { relative } from 'path'; export function lazyCompilation(server: DevServer) { const compiler = server.getCompiler(); @@ -16,14 +17,22 @@ export function lazyCompilation(server: DevServer) { if (ctx.path === '/__lazy_compile') { const paths = (ctx.query.paths as string).split(','); - - server.logger.info(`Lazy compiling ${chalk.cyan(paths.join(', '))}...`); + const pathsStr = paths + .map((p) => { + const resolvedPath = compiler.transformModulePath( + compiler.config.config.root, + p + ); + return relative(compiler.config.config.root, resolvedPath); + }) + .join(', '); + server.logger.info(`Lazy compiling ${chalk.cyan(pathsStr)}...`); const start = Date.now(); const result = await compiler.update(paths); server.logger.info( - `Lazy compilation done for ${chalk.cyan( - paths.join(', ') - )} in ${chalk.green(`${Date.now() - start}ms`)}.` + `Lazy compilation done for ${chalk.cyan(pathsStr)} in ${chalk.green( + `${Date.now() - start}ms` + )}.` ); if (result) { diff --git a/packages/core/src/watcher/index.ts b/packages/core/src/watcher/index.ts index c1a7db45d..69487053a 100644 --- a/packages/core/src/watcher/index.ts +++ b/packages/core/src/watcher/index.ts @@ -1,27 +1,59 @@ import chokidar, { FSWatcher } from 'chokidar'; +import path from 'path'; import { Compiler } from '../compiler/index.js'; import { DevServer } from '../server/index.js'; +export interface FileWatcherOptions { + ignores?: string[]; +} + export class FileWatcher { private _root: string; private _watcher: FSWatcher; + private _options: FileWatcherOptions; - constructor(root: string, config?: { ignores?: string[] }) { + constructor(root: string, config?: FileWatcherOptions) { this._root = root; - - this._watcher = chokidar.watch(`${this._root.replace('\\', '/')}/**`, { - ignored: config?.ignores ?? [], - }); + this._options = config ?? {}; } watch(serverOrCompiler: DevServer | Compiler) { + const compiler = + serverOrCompiler instanceof DevServer + ? serverOrCompiler.getCompiler() + : serverOrCompiler; + + this._watcher = chokidar.watch(compiler.resolvedModulePaths(this._root), { + ignored: this._options.ignores, + }); + + if (serverOrCompiler instanceof DevServer) { + serverOrCompiler.hmrEngine?.onUpdate((updateResult) => { + updateResult.added.forEach((addedModule) => { + const resolvedPath = compiler.transformModulePath( + this._root, + addedModule + ); + this._watcher.add(path.join(this._root, resolvedPath)); + }); + updateResult.removed.forEach((removedModule) => { + const resolvedPath = compiler.transformModulePath( + this._root, + removedModule + ); + this._watcher.unwatch(path.join(this._root, resolvedPath)); + }); + }); + } + this._watcher.on('change', (path) => { if (serverOrCompiler instanceof DevServer) { serverOrCompiler.hmrEngine.hmrUpdate(path); } else { // TODO update and emit the result - serverOrCompiler.updateSync([path]); + compiler.updateSync([path]); + compiler.writeResourcesToDisk(); } }); } diff --git a/packages/runtime-plugin-hmr/CHANGELOG.md b/packages/runtime-plugin-hmr/CHANGELOG.md index 3185bfd45..938b198c8 100644 --- a/packages/runtime-plugin-hmr/CHANGELOG.md +++ b/packages/runtime-plugin-hmr/CHANGELOG.md @@ -1,5 +1,11 @@ # @farmfe/runtime-plugin-hmr +## 3.0.5 + +### Patch Changes + +- limit the watched files to optimize cold start speed and fix lazy compilation issue" + ## 3.0.4 ### Patch Changes diff --git a/packages/runtime-plugin-hmr/package.json b/packages/runtime-plugin-hmr/package.json index edc528e89..93e596794 100644 --- a/packages/runtime-plugin-hmr/package.json +++ b/packages/runtime-plugin-hmr/package.json @@ -1,6 +1,6 @@ { "name": "@farmfe/runtime-plugin-hmr", - "version": "3.0.4", + "version": "3.0.5", "description": "Runtime hmr plugin of Farm", "author": { "name": "bright wu", @@ -11,6 +11,6 @@ "build": "tsc -p tsconfig.json --noEmit" }, "devDependencies": { - "@farmfe/runtime": "workspace:^0.3.2" + "@farmfe/runtime": "workspace:^0.3.3" } } diff --git a/packages/runtime-plugin-hmr/src/index.ts b/packages/runtime-plugin-hmr/src/index.ts index e747779be..d8ff81c81 100644 --- a/packages/runtime-plugin-hmr/src/index.ts +++ b/packages/runtime-plugin-hmr/src/index.ts @@ -24,7 +24,6 @@ export default { // the client will use the id to fetch the update resource and apply the update socket.addEventListener('message', (event) => { const data = JSON.parse(event.data) as HmrUpdatePacket; - import(`/__hmr?id=${data.id}`).then( (result: { default: HmrUpdateResult }) => { applyHotUpdates(result.default, moduleSystem); @@ -35,8 +34,8 @@ export default { socket.addEventListener('open', () => { console.log('[Farm HMR] connected to the server'); }); - - socket.addEventListener('close', () => setTimeout(connect, 3000)); + // TODO use ping/pong to detect the connection is closed, and if the server is online again, reload the page + // socket.addEventListener('close', () => setTimeout(connect, 3000)); return socket; } diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index c8d4ba82e..a88eb9daf 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -1,5 +1,11 @@ # @farmfe/runtime +## 0.3.3 + +### Patch Changes + +- limit the watched files to optimize cold start speed and fix lazy compilation issue" + ## 0.3.2 ### Patch Changes diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 08534f5d3..315a2694e 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@farmfe/runtime", - "version": "0.3.2", + "version": "0.3.3", "description": "Runtime of Farm", "author": { "name": "bright wu", diff --git a/packages/runtime/src/module-system.ts b/packages/runtime/src/module-system.ts index a8eca109a..5e1331a6f 100644 --- a/packages/runtime/src/module-system.ts +++ b/packages/runtime/src/module-system.ts @@ -76,7 +76,13 @@ export class ModuleSystem { dynamicRequire(moduleId: string): Promise { if (this.modules[moduleId]) { - return Promise.resolve(this.require(moduleId)); + const exports = this.require(moduleId); + + if (exports.__farm_async) { + return exports.default; + } else { + return Promise.resolve(exports); + } } const resources = this.dynamicModuleResourcesMap[moduleId]; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index de64c5ee7..a315a8233 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -92,8 +92,8 @@ importers: packages/core: specifiers: - '@farmfe/runtime': workspace:^0.3.2 - '@farmfe/runtime-plugin-hmr': workspace:^3.0.4 + '@farmfe/runtime': workspace:^0.3.3 + '@farmfe/runtime-plugin-hmr': workspace:^3.0.5 '@napi-rs/cli': ^2.10.0 '@swc/helpers': ^0.4.9 '@types/figlet': ^1.5.5 @@ -149,7 +149,7 @@ importers: packages/runtime-plugin-hmr: specifiers: - '@farmfe/runtime': workspace:^0.3.2 + '@farmfe/runtime': workspace:^0.3.3 devDependencies: '@farmfe/runtime': link:../runtime