feat: App builder bin 5.0

.changeset/brave-rats-tap.md

@@ -0,0 +1,6 @@
+---
+"app-builder-lib": minor
+"builder-util": minor
+---
+
+feat: update app-builder-bin to 5.0-alpha release

packages/app-builder-lib/src/packager.ts

@@ -21,7 +21,7 @@ import { ProtonFramework } from "./ProtonFramework"
 import { computeArchToTargetNamesMap, createTargets, NoOpTarget } from "./targets/targetFactory"
 import { computeDefaultAppDirectory, getConfig, validateConfiguration } from "./util/config"
 import { expandMacro } from "./util/macroExpander"
-import { createLazyProductionDeps, NodeModuleDirInfo } from "./util/packageDependencies"
+import { createLazyProductionDeps, NodeModuleDirInfo, NodeModuleInfo } from "./util/packageDependencies"
 import { checkMetadata, readPackageJson } from "./util/packageMetadata"
 import { getRepositoryInfo } from "./util/repositoryInfo"
 import { installOrRebuild, nodeGypRebuild } from "./util/yarn"
@@ -119,7 +119,7 @@ export class Packager {
 
   private nodeDependencyInfo = new Map<string, Lazy<Array<any>>>()
 
-  getNodeDependencyInfo(platform: Platform | null): Lazy<Array<NodeModuleDirInfo>> {
+  getNodeDependencyInfo(platform: Platform | null): Lazy<Array<NodeModuleInfo | NodeModuleDirInfo>> {
     let key = ""
     let excludedDependencies: Array<string> | null = null
     if (platform != null && this.framework.getExcludedDependencies != null) {
@@ -529,7 +529,7 @@ export class Packager {
         frameworkInfo,
         platform: platform.nodeName,
         arch: Arch[arch],
-        productionDeps: this.getNodeDependencyInfo(null),
+        productionDeps: this.getNodeDependencyInfo(null) as Lazy<Array<NodeModuleDirInfo>>,
       })
     }
   }

packages/app-builder-lib/src/util/NodeModuleCopyHelper.ts

@@ -6,6 +6,7 @@ import { excludedNames, FileMatcher } from "../fileMatcher"
 import { Packager } from "../packager"
 import { resolveFunction } from "../platformPackager"
 import { FileCopyHelper } from "./AppFileWalker"
+import { NodeModuleInfo } from "./packageDependencies"
 
 const excludedFiles = new Set(
   [".DS_Store", "node_modules" /* already in the queue */, "CHANGELOG.md", "ChangeLog", "changelog.md", "Changelog.md", "Changelog", "binding.gyp", ".npmignore"].concat(
@@ -37,108 +38,107 @@ export class NodeModuleCopyHelper extends FileCopyHelper {
     super(matcher, matcher.isEmpty() ? null : matcher.createFilter(), packager)
   }
 
-  async collectNodeModules(baseDir: string, moduleNames: Iterable<string>, nodeModuleExcludedExts: Array<string>): Promise<Array<string>> {
+  async collectNodeModules(moduleInfo: NodeModuleInfo, nodeModuleExcludedExts: Array<string>): Promise<Array<string>> {
     const filter = this.filter
     const metadata = this.metadata
 
     const onNodeModuleFile = await resolveFunction(this.packager.appInfo.type, this.packager.config.onNodeModuleFile, "onNodeModuleFile")
 
     const result: Array<string> = []
     const queue: Array<string> = []
-    for (const moduleName of moduleNames) {
-      const tmpPath = baseDir + path.sep + moduleName
-      queue.length = 1
-      // The path should be corrected in Windows that when the moduleName is Scoped packages named.
-      const depPath = path.normalize(tmpPath)
-      queue[0] = depPath
-
-      while (queue.length > 0) {
-        const dirPath = queue.pop()!
-
-        const childNames = await readdir(dirPath)
-        childNames.sort()
-
-        const isTopLevel = dirPath === depPath
-        const dirs: Array<string> = []
-        // our handler is async, but we should add sorted files, so, we add file to result not in the mapper, but after map
-        const sortedFilePaths = await BluebirdPromise.map(
-          childNames,
-          name => {
-            const filePath = dirPath + path.sep + name
-
-            const forceIncluded = onNodeModuleFile != null && !!onNodeModuleFile(filePath)
-
-            if (excludedFiles.has(name) || name.startsWith("._")) {
-              return null
-            }
-
-            if (!forceIncluded) {
-              for (const ext of nodeModuleExcludedExts) {
-                if (name.endsWith(ext)) {
-                  return null
-                }
-              }
+    const tmpPath = moduleInfo.dir
+    const moduleName = moduleInfo.name
+    queue.length = 1
+    // The path should be corrected in Windows that when the moduleName is Scoped packages named.
+    const depPath = path.normalize(tmpPath)
+    queue[0] = depPath
+
+    while (queue.length > 0) {
+      const dirPath = queue.pop()!
+
+      const childNames = await readdir(dirPath)
+      childNames.sort()
+
+      const isTopLevel = dirPath === depPath
+      const dirs: Array<string> = []
+      // our handler is async, but we should add sorted files, so, we add file to result not in the mapper, but after map
+      const sortedFilePaths = await BluebirdPromise.map(
+        childNames,
+        name => {
+          const filePath = dirPath + path.sep + name
+
+          const forceIncluded = onNodeModuleFile != null && !!onNodeModuleFile(filePath)
+
+          if (excludedFiles.has(name) || name.startsWith("._")) {
+            return null
+          }
 
-              // noinspection SpellCheckingInspection
-              if (isTopLevel && (topLevelExcludedFiles.has(name) || (moduleName === "libui-node" && (name === "build" || name === "docs" || name === "src")))) {
+          if (!forceIncluded) {
+            for (const ext of nodeModuleExcludedExts) {
+              if (name.endsWith(ext)) {
                 return null
               }
+            }
 
-              if (dirPath.endsWith("build")) {
-                if (name === "gyp-mac-tool" || name === "Makefile" || name.endsWith(".mk") || name.endsWith(".gypi") || name.endsWith(".Makefile")) {
-                  return null
-                }
-              } else if (dirPath.endsWith("Release") && (name === ".deps" || name === "obj.target")) {
-                return null
-              } else if (name === "src" && (dirPath.endsWith("keytar") || dirPath.endsWith("keytar-prebuild"))) {
-                return null
-              } else if (dirPath.endsWith("lzma-native") && (name === "build" || name === "deps")) {
-                return null
-              }
+            // noinspection SpellCheckingInspection
+            if (isTopLevel && (topLevelExcludedFiles.has(name) || (moduleName === "libui-node" && (name === "build" || name === "docs" || name === "src")))) {
+              return null
             }
 
-            return lstat(filePath).then(stat => {
-              if (filter != null && !filter(filePath, stat)) {
+            if (dirPath.endsWith("build")) {
+              if (name === "gyp-mac-tool" || name === "Makefile" || name.endsWith(".mk") || name.endsWith(".gypi") || name.endsWith(".Makefile")) {
                 return null
               }
+            } else if (dirPath.endsWith("Release") && (name === ".deps" || name === "obj.target")) {
+              return null
+            } else if (name === "src" && (dirPath.endsWith("keytar") || dirPath.endsWith("keytar-prebuild"))) {
+              return null
+            } else if (dirPath.endsWith("lzma-native") && (name === "build" || name === "deps")) {
+              return null
+            }
+          }
 
-              if (!stat.isDirectory()) {
-                metadata.set(filePath, stat)
+          return lstat(filePath).then(stat => {
+            if (filter != null && !filter(filePath, stat)) {
+              return null
+            }
+
+            if (!stat.isDirectory()) {
+              metadata.set(filePath, stat)
+            }
+            const consumerResult = this.handleFile(filePath, dirPath, stat)
+            if (consumerResult == null) {
+              if (stat.isDirectory()) {
+                dirs.push(name)
+                return null
+              } else {
+                return filePath
               }
-              const consumerResult = this.handleFile(filePath, dirPath, stat)
-              if (consumerResult == null) {
-                if (stat.isDirectory()) {
+            } else {
+              return consumerResult.then(it => {
+                // asarUtil can return modified stat (symlink handling)
+                if ((it == null ? stat : it).isDirectory()) {
                   dirs.push(name)
                   return null
                 } else {
                   return filePath
                 }
-              } else {
-                return consumerResult.then(it => {
-                  // asarUtil can return modified stat (symlink handling)
-                  if ((it == null ? stat : it).isDirectory()) {
-                    dirs.push(name)
-                    return null
-                  } else {
-                    return filePath
-                  }
-                })
-              }
-            })
-          },
-          CONCURRENCY
-        )
-
-        for (const child of sortedFilePaths) {
-          if (child != null) {
-            result.push(child)
-          }
+              })
+            }
+          })
+        },
+        CONCURRENCY
+      )
+
+      for (const child of sortedFilePaths) {
+        if (child != null) {
+          result.push(child)
         }
+      }
 
-        dirs.sort()
-        for (const child of dirs) {
-          queue.push(dirPath + path.sep + child)
-        }
+      dirs.sort()
+      for (const child of dirs) {
+        queue.push(dirPath + path.sep + child)
       }
     }
     return result

packages/app-builder-lib/src/util/appFileCopier.ts

@@ -13,11 +13,37 @@ import { Packager } from "../packager"
 import { PlatformPackager } from "../platformPackager"
 import { AppFileWalker } from "./AppFileWalker"
 import { NodeModuleCopyHelper } from "./NodeModuleCopyHelper"
+import { NodeModuleInfo } from "./packageDependencies"
 
 const BOWER_COMPONENTS_PATTERN = `${path.sep}bower_components${path.sep}`
 /** @internal */
 export const ELECTRON_COMPILE_SHIM_FILENAME = "__shim.js"
 
+function extractPathAfterLastNodeModules(src: string, file: string) {
+  const srcComponents = src.split(path.sep)
+  const lastNodeModulesIndex = srcComponents.lastIndexOf("node_modules")
+
+  if (lastNodeModulesIndex === -1 || lastNodeModulesIndex === srcComponents.length - 1) {
+    return ""
+  }
+
+  const pathAfterNodeModules = srcComponents.slice(lastNodeModulesIndex + 1).join(path.sep)
+
+  const matchIndex = file.indexOf(pathAfterNodeModules)
+
+  if (matchIndex === -1) {
+    return ""
+  }
+
+  const remainingPathStartIndex = matchIndex + pathAfterNodeModules.length
+  if (remainingPathStartIndex >= file.length) {
+    return ""
+  }
+
+  const remainingPath = file.substring(remainingPathStartIndex).trim()
+  return remainingPath.startsWith(path.sep) ? remainingPath.substring(1) : remainingPath
+}
+
 export function getDestinationPath(file: string, fileSet: ResolvedFileSet) {
   if (file === fileSet.src) {
     return fileSet.destination
@@ -27,17 +53,8 @@ export function getDestinationPath(file: string, fileSet: ResolvedFileSet) {
     if (file.length > src.length && file.startsWith(src) && file[src.length] === path.sep) {
       return dest + file.substring(src.length)
     } else {
-      // hoisted node_modules
-      // not lastIndexOf, to ensure that nested module (top-level module depends on) copied to parent node_modules, not to top-level directory
-      // project https://github.com/angexis/punchcontrol/commit/cf929aba55c40d0d8901c54df7945e1d001ce022
-      let index = file.indexOf(NODE_MODULES_PATTERN)
-      if (index < 0 && file.endsWith(`${path.sep}node_modules`)) {
-        index = file.length - 13
-      }
-      if (index < 0) {
-        throw new Error(`File "${file}" not under the source directory "${fileSet.src}"`)
-      }
-      return dest + file.substring(index)
+      const e = extractPathAfterLastNodeModules(src, file)
+      return path.join(dest, e)
     }
   }
 }
@@ -191,25 +208,38 @@ function validateFileSet(fileSet: ResolvedFileSet): ResolvedFileSet {
 
 /** @internal */
 export async function computeNodeModuleFileSets(platformPackager: PlatformPackager<any>, mainMatcher: FileMatcher): Promise<Array<ResolvedFileSet>> {
-  const deps = await platformPackager.info.getNodeDependencyInfo(platformPackager.platform).value
+  const deps = (await platformPackager.info.getNodeDependencyInfo(platformPackager.platform).value) as Array<NodeModuleInfo>
+
   const nodeModuleExcludedExts = getNodeModuleExcludedExts(platformPackager)
   // serial execution because copyNodeModules is concurrent and so, no need to increase queue/pressure
   const result = new Array<ResolvedFileSet>()
   let index = 0
   for (const info of deps) {
-    const source = info.dir
+    const source = platformPackager.info.appDir + path.sep + "node_modules" + path.sep + info.name
+    // const source = platformPackager.info.appDir
     const destination = getDestinationPath(source, { src: mainMatcher.from, destination: mainMatcher.to, files: [], metadata: null as any })
 
     // use main matcher patterns, so, user can exclude some files in such hoisted node modules
     // source here includes node_modules, but pattern base should be without because users expect that pattern "!node_modules/loot-core/src{,/**/*}" will work
-    const matcher = new FileMatcher(path.dirname(source), destination, mainMatcher.macroExpander, mainMatcher.patterns)
+    const matcher = new FileMatcher(path.dirname(platformPackager.info.appDir + path.sep + "node_modules"), destination, mainMatcher.macroExpander, mainMatcher.patterns)
     const copier = new NodeModuleCopyHelper(matcher, platformPackager.info)
-    const files = await copier.collectNodeModules(
-      source,
-      info.deps.map(it => it.name),
-      nodeModuleExcludedExts
-    )
+    const files = await copier.collectNodeModules(info, nodeModuleExcludedExts)
     result[index++] = validateFileSet({ src: source, destination, files, metadata: copier.metadata })
+
+    if (info.conflictDependency) {
+      for (const dep of info.conflictDependency) {
+        const source = platformPackager.info.appDir + path.sep + "node_modules" + path.sep + info.name + path.sep + "node_modules" + path.sep + dep.name
+        const destination = getDestinationPath(source, { src: mainMatcher.from, destination: mainMatcher.to, files: [], metadata: null as any })
+        const matcher = new FileMatcher(
+          path.dirname(platformPackager.info.appDir + path.sep + "node_modules" + path.sep + info.name + path.sep + "node_modules"),
+          destination,
+          mainMatcher.macroExpander,
+          mainMatcher.patterns
+        )
+        const copier = new NodeModuleCopyHelper(matcher, platformPackager.info)
+        result[index++] = validateFileSet({ src: source, destination, files: await copier.collectNodeModules(dep, nodeModuleExcludedExts), metadata: copier.metadata })
+      }
+    }
   }
   return result
 }

packages/app-builder-lib/src/util/packageDependencies.ts

@@ -3,7 +3,7 @@ import { executeAppBuilderAsJson } from "./appBuilder"
 
 export function createLazyProductionDeps(projectDir: string, excludedDependencies: Array<string> | null) {
   return new Lazy(async () => {
-    const args = ["node-dep-tree", "--dir", projectDir]
+    const args = ["node-dep-tree", "--flatten", "--dir", projectDir]
     if (excludedDependencies != null) {
       for (const name of excludedDependencies) {
         args.push("--exclude-dep", name)
@@ -20,4 +20,7 @@ export interface NodeModuleDirInfo {
 
 export interface NodeModuleInfo {
   readonly name: string
+  readonly version: string
+  readonly dir: string
+  readonly conflictDependency: Array<NodeModuleInfo>
 }

packages/builder-util/package.json

@@ -17,7 +17,7 @@
   "dependencies": {
     "7zip-bin": "~5.2.0",
     "@types/debug": "^4.1.6",
-    "app-builder-bin": "4.0.0",
+    "app-builder-bin": "v5.0.0-alpha.3",
     "bluebird-lst": "^1.0.9",
     "builder-util-runtime": "workspace:*",
     "chalk": "^4.1.2",