feat: add NodeModulesLayer construct to enable faster uploads in watc…

…h-mode, fixing #15 (#16)

* feat: add NodeModulesLayer construct for fixing #15

* fix: test types

package.json

@@ -11,7 +11,8 @@
   "version": "0.0.19-next.2",
   "scripts": {
     "type-check": "tsc -p tsconfig.json --noEmit",
-    "build": "rm -rf lib && tsc -p tsconfig.json && chmod +x ./lib/lambda-extension/cdk-watch-lambda-wrapper/index.js",
+    "build": "tsup src/index.ts src/cli.ts src/index.ts src/websocketHandlers/index.ts src/lambda-extension/cdk-watch-lambda-wrapper/index.ts --no-splitting -d lib --clean --dts=src/index.ts",
+    "watch": "yarn build --watch",
     "lint": "eslint ./src --ext=.ts",
     "try": "node -r ts-node/register src/cli.ts",
     "postinstall": "husky install",
@@ -85,6 +86,7 @@
     "standard-version": "^9.1.0",
     "ts-jest": "^26.5.1",
     "ts-node": "^9.1.1",
+    "tsup": "^4.8.19",
     "typescript": "~4.0.0"
   },
   "prettier": "prettier-config-planes",

src/cli.ts

@@ -1,7 +1,13 @@
 #!/usr/bin/env node
+import * as fs from 'fs-extra';
+import * as path from 'path';
 import {CdkWatchCommand} from './commands';
 
-const program = new CdkWatchCommand();
+// NOTE: When this entry is built it's bundled into `/lib/cli.js`. So this is
+// relative to that path.
+const {version} = fs.readJSONSync(path.resolve(__dirname, '../package.json'));
+
+const program = new CdkWatchCommand(version);
 
 program.parseAsync(process.argv).catch((e) => {
   // eslint-disable-next-line no-console

src/commands/__tests__/commands.test.ts

@@ -17,7 +17,7 @@ describe('CLI commands', () => {
   });
 
   test('program runs watch as the default command', async () => {
-    const program = new CdkWatchCommand();
+    const program = new CdkWatchCommand('1');
     await program.parseAsync(buildArgv('**'));
     expect(watch).toBeCalledWith('**', {logs: true}, expect.anything());
     expect(list).toBeCalledTimes(0);
@@ -26,7 +26,7 @@ describe('CLI commands', () => {
   });
 
   test('program runs watch when command name is provided', async () => {
-    const program = new CdkWatchCommand();
+    const program = new CdkWatchCommand('1');
     await program.parseAsync(buildArgv('watch My/Path'));
     expect(watch).toBeCalledWith('My/Path', {logs: true}, expect.anything());
     expect(list).toBeCalledTimes(0);
@@ -43,7 +43,7 @@ describe('CLI commands', () => {
   `(
     'command runs correct function',
     async ({command}: {command: keyof typeof otherCommands}) => {
-      const program = new CdkWatchCommand();
+      const program = new CdkWatchCommand('1');
       await program.parseAsync(buildArgv(`${command} My/Path`));
       expect(otherCommands[command]).toBeCalledWith(
         'My/Path',
@@ -60,7 +60,7 @@ describe('CLI commands', () => {
   `(
     'logs are on by default but can be turned off',
     async ({flag, expected}) => {
-      const program = new CdkWatchCommand();
+      const program = new CdkWatchCommand('1');
       await program.parseAsync(buildArgv(`watch My/Path ${flag}`));
       expect(watch).toBeCalledWith(
         'My/Path',

src/commands/index.ts

@@ -1,17 +1,13 @@
 import {Command, Option} from 'commander';
-import * as fs from 'fs-extra';
-import * as path from 'path';
 import {list} from './list';
 import {logs} from './logs';
 import {once} from './once';
 import {watch} from './watch';
 
 class CdkWatchCommand extends Command {
-  constructor() {
+  constructor(version: string) {
     super();
-    const {version} = fs.readJSONSync(
-      path.resolve(__dirname, '../../package.json'),
-    );
+
     this.version(version);
 
     const profileOption = new Option(

src/constructs/LogsLayerVersion.ts

@@ -8,7 +8,8 @@ export class LogsLayerVersion extends LayerVersion {
       removalPolicy: RemovalPolicy.DESTROY,
       description:
         'Catches Lambda Logs and sends them to API Gateway Connections',
-      code: Code.fromAsset(path.join(__dirname, '../', 'lambda-extension')),
+      // NOTE: This file will be bundled into /lib/index.js, so this path must be relative to that
+      code: Code.fromAsset(path.join(__dirname, 'lambda-extension')),
     });
   }
 }

src/constructs/NodeModulesLayer.ts

@@ -0,0 +1,94 @@
+import {Code, LayerVersion, LayerVersionProps} from '@aws-cdk/aws-lambda';
+import {
+  extractDependencies,
+  findUp,
+  LockFile,
+} from '@aws-cdk/aws-lambda-nodejs/lib/util';
+import {Construct, RemovalPolicy} from '@aws-cdk/core';
+import execa from 'execa';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import {CDK_WATCH_OUTDIR} from '../consts';
+
+interface NodeModulesLayerProps {
+  depsLockFilePath?: string;
+  pkgPath: string;
+  nodeModules: string[];
+}
+
+enum Installer {
+  NPM = 'npm',
+  YARN = 'yarn',
+}
+
+/**
+ * Copied from cdk source:
+ * https://github.com/aws/aws-cdk/blob/ca42461acd4f42a8bd7c0fb05788c7ea50834de2/packages/@aws-cdk/aws-lambda-nodejs/lib/function.ts#L88-L103
+ */
+const getDepsLock = (propsDepsLockFilePath?: string) => {
+  let depsLockFilePath: string;
+  if (propsDepsLockFilePath) {
+    if (!fs.existsSync(propsDepsLockFilePath)) {
+      throw new Error(`Lock file at ${propsDepsLockFilePath} doesn't exist`);
+    }
+    if (!fs.statSync(propsDepsLockFilePath).isFile()) {
+      throw new Error('`depsLockFilePath` should point to a file');
+    }
+    depsLockFilePath = path.resolve(propsDepsLockFilePath);
+  } else {
+    const lockFile = findUp(LockFile.YARN) ?? findUp(LockFile.NPM);
+    if (!lockFile) {
+      throw new Error(
+        'Cannot find a package lock file (`yarn.lock` or `package-lock.json`). Please specify it with `depsFileLockPath`.',
+      );
+    }
+    depsLockFilePath = lockFile;
+  }
+
+  return depsLockFilePath;
+};
+
+export class NodeModulesLayer extends LayerVersion {
+  constructor(scope: Construct, id: string, props: NodeModulesLayerProps) {
+    const depsLockFilePath = getDepsLock(props.depsLockFilePath);
+
+    const {pkgPath} = props;
+
+    // Determine dependencies versions, lock file and installer
+    const dependencies = extractDependencies(pkgPath, props.nodeModules);
+    let installer = Installer.NPM;
+    let lockFile = LockFile.NPM;
+    if (depsLockFilePath.endsWith(LockFile.YARN)) {
+      lockFile = LockFile.YARN;
+      installer = Installer.YARN;
+    }
+
+    const layerBase = path.join(
+      process.cwd(),
+      'cdk.out',
+      CDK_WATCH_OUTDIR,
+      'node-module-layers',
+      scope.node.addr,
+    );
+    const outputDir = path.join(layerBase, 'nodejs');
+
+    fs.ensureDirSync(outputDir);
+    fs.copyFileSync(depsLockFilePath, path.join(outputDir, lockFile));
+    fs.writeJsonSync(path.join(outputDir, 'package.json'), {dependencies});
+
+    // eslint-disable-next-line no-console
+    console.log('Installing node_modules in layer');
+    execa.sync(installer, ['install'], {
+      cwd: outputDir,
+      stderr: 'inherit',
+      stdout: 'ignore',
+      stdin: 'ignore',
+    });
+
+    super(scope, id, {
+      removalPolicy: RemovalPolicy.DESTROY,
+      description: 'NodeJS Modules Packaged into a Layer by cdk-watch',
+      code: Code.fromAsset(layerBase),
+    });
+  }
+}

src/constructs/RealTimeLambdaLogsAPI.ts

@@ -36,11 +36,8 @@ export class RealTimeLambdaLogsAPI extends cdk.NestedStack {
 
     const stack = Stack.of(this);
     const routeSelectionKey = 'action';
-    const websocketHandlerCodePath = path.join(
-      __dirname,
-      '..',
-      'websocketHandlers',
-    );
+    // NOTE: This file will be bundled into /lib/index.js, so this path must be relative to that
+    const websocketHandlerCodePath = path.join(__dirname, 'websocketHandlers');
 
     this.logsLayerVersion = new LogsLayerVersion(this, 'LogsLayerVersion');
 

src/constructs/WatchableNodejsFunction.ts

@@ -1,9 +1,14 @@
 /* eslint-disable no-new */
 /* eslint-disable import/no-extraneous-dependencies */
-import {NodejsFunction, NodejsFunctionProps} from '@aws-cdk/aws-lambda-nodejs';
+import {
+  BundlingOptions,
+  NodejsFunction,
+  NodejsFunctionProps,
+} from '@aws-cdk/aws-lambda-nodejs';
 import {Runtime} from '@aws-cdk/aws-lambda';
 import {Asset} from '@aws-cdk/aws-s3-assets';
 import * as path from 'path';
+import * as fs from 'fs-extra';
 import findUp from 'find-up';
 import * as cdk from '@aws-cdk/core';
 import {BuildOptions, Loader} from 'esbuild';
@@ -12,8 +17,26 @@ import {readManifest} from '../lib/readManifest';
 import {writeManifest} from '../lib/writeManifest';
 import {RealTimeLambdaLogsAPI} from './RealTimeLambdaLogsAPI';
 import {CDK_WATCH_CONTEXT_LOGS_ENABLED} from '../consts';
+import {NodeModulesLayer} from './NodeModulesLayer';
 
+interface WatchableBundlingOptions extends BundlingOptions {
+  /**
+   * Similar to `bundling.nodeModules` however in this case your modules will be
+   * bundled into a Lambda layer instead of being uploaded with your lambda
+   * function code. This has upside when 'watching' your code as the only code
+   * that needs to be uploaded each time is your core lambda code rather than
+   * any modules, which are unlikely to change frequently. Passing `true` will
+   * load all modules found in the "dependencies" of the entries package.json
+   */
+  nodeModulesLayer?: boolean | string[];
+}
+
+// NodeModulesLayer
 interface WatchableNodejsFunctionProps extends NodejsFunctionProps {
+  /**
+   * Bundling options.
+   */
+  bundling?: WatchableBundlingOptions;
   /**
    * CDK Watch Options
    */
@@ -37,37 +60,90 @@ class WatchableNodejsFunction extends NodejsFunction {
 
   public cdkWatchLogsApi?: RealTimeLambdaLogsAPI;
 
+  public readonly local?: cdk.ILocalBundling;
+
   constructor(
     scope: cdk.Construct,
     id: string,
     props: WatchableNodejsFunctionProps,
   ) {
-    super(scope, id, props);
+    if (!props.entry) throw new Error('Expected props.entry');
+    const pkgPath = findUp.sync('package.json', {
+      cwd: path.dirname(props.entry),
+    });
+    if (!pkgPath) {
+      throw new Error(
+        'Cannot find a `package.json` in this project. Using `nodeModules` requires a `package.json`.',
+      );
+    }
+    const nodeModulesLayerOption = props.bundling?.nodeModulesLayer;
+    const shouldCreateModulesLayer =
+      typeof nodeModulesLayerOption === 'boolean'
+        ? nodeModulesLayerOption
+        : (nodeModulesLayerOption?.length ?? 0) > 0;
+
+    let nodeModulesLayer: null | NodeModulesLayer = null;
+    let moduleNames: string[] = [];
+    if (shouldCreateModulesLayer && nodeModulesLayerOption) {
+      if (typeof nodeModulesLayerOption === 'boolean') {
+        const packageJson = fs.readJSONSync(pkgPath);
+        moduleNames = Object.keys(packageJson.dependencies || {});
+      } else {
+        moduleNames = nodeModulesLayerOption;
+      }
+      nodeModulesLayer = new NodeModulesLayer(scope, 'NodeModulesLayer', {
+        pkgPath,
+        nodeModules: moduleNames,
+        depsLockFilePath: props.depsLockFilePath,
+      });
+    }
+    const bundling: WatchableBundlingOptions = {
+      ...props.bundling,
+      externalModules: [
+        ...moduleNames,
+        ...(props.bundling?.externalModules || ['aws-sdk']),
+      ],
+    };
+    super(scope, id, {
+      ...props,
+      bundling,
+    });
+
+    if (nodeModulesLayer) {
+      this.addLayers(nodeModulesLayer);
+    }
+
     const {entry} = props;
     if (!entry) throw new Error('`entry` must be provided');
-    const target = props.runtime?.runtimeEquals(Runtime.NODEJS_10_X)
-      ? 'node10'
-      : 'node12';
+    const targetMatch = (props.runtime || Runtime.NODEJS_12_X).name.match(
+      /nodejs(\d+)/,
+    );
+    if (!targetMatch) {
+      throw new Error('Cannot extract version from runtime.');
+    }
+    const target = `node${targetMatch[1]}`;
+
     this.esbuildOptions = {
       target,
       bundle: true,
       entryPoints: [entry],
       platform: 'node',
-      minify: props.bundling?.minify ?? false,
-      sourcemap: props.bundling?.sourceMap,
+      minify: bundling?.minify ?? false,
+      sourcemap: bundling?.sourceMap,
       external: [
-        ...(props.bundling?.externalModules ?? ['aws-sdk']),
-        ...(props.bundling?.nodeModules ?? []),
+        ...(bundling?.externalModules ?? ['aws-sdk']),
+        ...(bundling?.nodeModules ?? []),
+        ...(moduleNames ?? []),
       ],
-      loader: props.bundling?.loader as {[ext: string]: Loader} | undefined,
-      define: props.bundling?.define,
-      logLevel: props.bundling?.logLevel,
-      keepNames: props.bundling?.keepNames,
-      tsconfig: props.bundling?.tsconfig
-        ? path.resolve(entry, path.resolve(props.bundling?.tsconfig))
+      loader: bundling?.loader as {[ext: string]: Loader} | undefined,
+      define: bundling?.define,
+      logLevel: bundling?.logLevel,
+      keepNames: bundling?.keepNames,
+      tsconfig: bundling?.tsconfig
+        ? path.resolve(entry, path.resolve(bundling?.tsconfig))
         : findUp.sync('tsconfig.json', {cwd: path.dirname(entry)}),
-      banner: props.bundling?.banner,
-      footer: props.bundling?.footer,
+      banner: bundling?.banner,
+      footer: bundling?.footer,
     };
 
     if (

src/lib/updateLambdaFunctionCode.ts

@@ -1,4 +1,4 @@
-import {AWSError, CloudFormation, Lambda} from 'aws-sdk';
+import {AWSError, Lambda} from 'aws-sdk';
 import {PromiseResult} from 'aws-sdk/lib/request';
 import {zipDirectory} from './zipDirectory';