Add support for Yarn workspaces (PoC for #189)

bin/node2nix.js

@@ -33,7 +33,8 @@ var switches = [
     ['--no-bypass-cache', 'Specifies that package builds do not need to bypass the content addressable cache (required for NPM 5.x)'],
     ['--no-copy-node-env', 'Do not create a copy of the Nix expression that builds NPM packages'],
     ['--use-fetchgit-private', 'Use fetchGitPrivate instead of fetchgit in the generated Nix expressions'],
-    ['--strip-optional-dependencies', 'Strips the optional dependencies from the regular dependencies in the NPM registry']
+    ['--strip-optional-dependencies', 'Strips the optional dependencies from the regular dependencies in the NPM registry'],
+    ['--yarn-workspace FILE', 'Use a Yarn workspace package.json to find project interdependencies'],
 ];
 
 var parser = new optparse.OptionParser(switches);
@@ -59,6 +60,7 @@ var noCopyNodeEnv = false;
 var bypassCache = true;
 var useFetchGitPrivate = false;
 var stripOptionalDependencies = false;
+var yarnWorkspaceJSON;
 var executable;
 
 /* Define process rules for option parameters */
@@ -185,6 +187,10 @@ parser.on('strip-optional-dependencies', function(arg, value) {
     stripOptionalDependencies = true;
 });
 
+parser.on('yarn-workspace', function(arg, value) {
+    yarnWorkspaceJSON = value;
+});
+
 /* Define process rules for non-option parameters */
 
 parser.on(1, function(opt) {
@@ -247,7 +253,7 @@ if(version) {
 }
 
 /* Perform the NPM to Nix conversion */
-node2nix.npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, registryAuthToken, noCopyNodeEnv, bypassCache, useFetchGitPrivate, stripOptionalDependencies, function(err) {
+node2nix.npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, registryAuthToken, noCopyNodeEnv, bypassCache, useFetchGitPrivate, stripOptionalDependencies, yarnWorkspaceJSON, function(err) {
     if(err) {
         process.stderr.write(err + "\n");
         process.exit(1);

lib/DeploymentConfig.js

@@ -27,6 +27,7 @@ function DeploymentConfig(registryURL, registryAuthToken, production, includePee
     this.outputDir = outputDir;
     this.bypassCache = bypassCache;
     this.stripOptionalDependencies = stripOptionalDependencies;
+    this.packageOverrides = {};
 }
 
 exports.DeploymentConfig = DeploymentConfig;

lib/Package.js

@@ -1,6 +1,5 @@
 var path = require('path');
 var slasp = require('slasp');
-var semver = require('semver');
 var nijs = require('nijs');
 var Source = require('./sources/Source.js').Source;
 var inherit = require('nijs/lib/ast/util/inherit.js').inherit;
@@ -64,7 +63,7 @@ Package.prototype.findMatchingProvidedDependencyByParent = function(name, versio
         } else if(dependency === null) {
              return null; // If we have encountered a bundled dependency with the same name, consider it a conflict (is not a perfect resolution, but does not result in an error)
         } else {
-            if(semver.satisfies(dependency.source.config.version, versionSpec, true)) { // If we found a dependency with the same name, see if the version fits
+            if(dependency.source.versionSatisfies(versionSpec)) { // If we found a dependency with the same name, see if the version fits
                 return dependency;
             } else {
                 return null; // If there is a version mismatch, then a conflicting version has been encountered
@@ -144,20 +143,27 @@ Package.prototype.bundleDependencies = function(resolvedDependencies, dependenci
         slasp.fromEach(function(callback) {
             callback(null, dependencies);
         }, function(dependencyName, callback) {
-            var versionSpec = dependencies[dependencyName];
-            var parentDependency = self.findMatchingProvidedDependencyByParent(dependencyName, versionSpec);
-
             if(self.isBundledDependency(dependencyName)) {
                 self.requiredDependencies[dependencyName] = null;
-                callback();
-            } else if(parentDependency === null) {
-                var pkg = new Package(self.deploymentConfig, self.lock, self, dependencyName, versionSpec, self.source.baseDir, true /* Never include development dependencies of transitive dependencies */, self.sourcesCache);
+                return callback();
+            }
 
+            var pkg = self.deploymentConfig.packageOverrides[dependencyName];
+            var bundlePkg = !!pkg;
+            if(!pkg) {
+                var versionSpec = dependencies[dependencyName];
+                pkg = self.findMatchingProvidedDependencyByParent(dependencyName, versionSpec);
+            }
+            if(!pkg) {
+                pkg = new Package(self.deploymentConfig, self.lock, self, dependencyName, versionSpec, self.source.baseDir, true /* Never include development dependencies of transitive dependencies */, self.sourcesCache);
+                bundlePkg = true;
+            }
+
+            if(bundlePkg) {
                 slasp.sequence([
                     function(callback) {
                         pkg.source.fetch(callback);
                     },
-
                     function(callback) {
                         self.sourcesCache.addSource(pkg.source);
                         self.bundleDependency(dependencyName, pkg);
@@ -166,7 +172,7 @@ Package.prototype.bundleDependencies = function(resolvedDependencies, dependenci
                     }
                 ], callback);
             } else {
-                self.requiredDependencies[dependencyName] = parentDependency; // If there is a parent package that provides the requested dependency -> use it
+                self.requiredDependencies[dependencyName] = pkg; // If there is a parent package that provides the requested dependency -> use it
                 callback();
             }
 

lib/expressions/OutputExpression.js

@@ -1,6 +1,7 @@
 var nijs = require('nijs');
 var inherit = require('nijs/lib/ast/util/inherit.js').inherit;
 var SourcesCache = require('../SourcesCache.js').SourcesCache;
+var WorkspaceSource = require('../sources/WorkspaceSource').WorkspaceSource;
 
 /**
  * Creates a new output expression instance.
@@ -34,13 +35,24 @@ OutputExpression.prototype.resolveDependencies = function(callback) {
  * @see NixASTNode#toNixAST
  */
 OutputExpression.prototype.toNixAST = function() {
+    var argSpec = {
+        nodeEnv: undefined,
+        fetchurl: undefined,
+        fetchgit: undefined,
+        globalBuildInputs: []
+    };
+
+    // Add arguments for workspace dependencies
+    for(var identifier in this.sourcesCache.sources) {
+        var source = this.sourcesCache.sources[identifier];
+        if(source instanceof WorkspaceSource && !source.symlink) {
+            var varName = source.variableName();
+            argSpec[varName] = source.fileExpression();
+        }
+    }
+
     return new nijs.NixFunction({
-        argSpec: {
-            nodeEnv: undefined,
-            fetchurl: undefined,
-            fetchgit: undefined,
-            globalBuildInputs: []
-        },
+        argSpec: argSpec,
         body: new nijs.NixLet({
             value: {
                 sources: this.sourcesCache

lib/node2nix.js

@@ -6,11 +6,13 @@ var fs = require('fs');
 var path = require('path');
 var slasp = require('slasp');
 var nijs = require('nijs');
+var glob = require('glob');
 
 var CollectionExpression = require('./expressions/CollectionExpression.js').CollectionExpression;
 var PackageExpression = require('./expressions/PackageExpression.js').PackageExpression;
 var CompositionExpression = require('./expressions/CompositionExpression.js').CompositionExpression;
 var DeploymentConfig = require('./DeploymentConfig.js').DeploymentConfig;
+var Package = require('./Package.js').Package;
 
 function copyNodeEnvExpr(nodeEnvNix, callback) {
     /* Compose a read stream that reads the build expression */
@@ -32,6 +34,59 @@ function copyNodeEnvExpr(nodeEnvNix, callback) {
     rs.pipe(ws);
 }
 
+/**
+ * Resolve a Yarn workspaces list to package paths.
+ *
+ * The result is a map of package names to paths relative to baseDir.
+ *
+ * @function
+ * @param {Array<String>} workspaces The workspaces field from package.json
+ * @param {String} baseDir Directory in which the referrer's package.json configuration resides
+ * @param {String} resolveDir Directory in which the workspace package.json configuration resides
+ * @param {function(String)} callback Callback that gets invoked when the work
+ *     is done. In case of an error, the first parameter contains a string with
+ *     an error message.
+ */
+function resolveYarnWorkspaces(workspaces, baseDir, resolveDir, callback) {
+    var pkgPaths = {};
+    slasp.sequence([
+        function(callback) {
+            // Resolve globs to unique paths relative to baseDir.
+            slasp.fromEach(function(callback) {
+                callback(null, workspaces);
+            }, function(idx, callback) {
+                slasp.sequence([
+                    function(callback) {
+                        glob(workspaces[idx] + '/package.json', {
+                            cwd: resolveDir,
+                            absolute: true,
+                        }, callback);
+                    },
+                    function(callback, matches) {
+                        matches.forEach(function(match) {
+                            var matchRel = path.relative(baseDir, match);
+                            if (matchRel !== 'package.json') { // ignore self
+                                var pkgPath = matchRel.slice(0, -13)
+                                pkgPaths[pkgPath] = true;
+                            }
+                        });
+                        callback();
+                    }
+                ], callback);
+            }, callback);
+        },
+        function(callback) {
+            // Build a map of: name -> path
+            var pkgsMap = {};
+            Object.keys(pkgPaths).forEach(function(pkgPath) {
+                var config = JSON.parse(fs.readFileSync(pkgPath + '/package.json'));
+                pkgsMap[config.name] = pkgPath;
+            });
+            callback(null, pkgsMap);
+        }
+    ], callback);
+}
+
 /**
  * Writes a copy of node-env.nix to a specified path.
  *
@@ -42,7 +97,7 @@ function copyNodeEnvExpr(nodeEnvNix, callback) {
  */
 exports.copyNodeEnvExpr = copyNodeEnvExpr;
 
-function npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, registryAuthToken, noCopyNodeEnv, bypassCache, useFetchGitPrivate, stripOptionalDependencies, callback) {
+function npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, registryAuthToken, noCopyNodeEnv, bypassCache, useFetchGitPrivate, stripOptionalDependencies, yarnWorkspaceJSON, callback) {
     var obj = JSON.parse(fs.readFileSync(inputJSON));
     var version = JSON.parse(fs.readFileSync(path.join(__dirname, "..", "package.json"))).version;
     var disclaimer = "# This file has been generated by node2nix " + version + ". Do not edit!\n\n";
@@ -66,6 +121,7 @@ function npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, su
             if(typeof obj == "object" && obj !== null) {
                 if(Array.isArray(obj)) {
                     expr = new CollectionExpression(deploymentConfig, baseDir, obj);
+                    callback();
                 } else {
                     // Display error if mandatory package.json attributes are not set
                     if(!obj.name) {
@@ -79,14 +135,68 @@ function npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, lockJSON, su
 
                     // Display a warning if we expect a lock file to be used, but the user does not specify it
                     displayLockWarning = bypassCache && !lockJSON && fs.existsSync(path.join(path.dirname(inputJSON), path.basename(inputJSON, ".json")) + "-lock.json");
-                }
 
-                expr.resolveDependencies(callback);
+                    // If this is a Yarn workspace, bundle all packages as virtual dependencies at the top level
+                    if(obj.workspaces) {
+                        slasp.sequence([
+                            function(callback) {
+                                resolveYarnWorkspaces(obj.workspaces, baseDir, baseDir, callback);
+                            },
+                            function(callback, pkgs) {
+                                slasp.fromEach(function(callback) {
+                                    callback(null, pkgs);
+                                }, function(pkgName, callback) {
+                                    var pkgPath = pkgs[pkgName];
+                                    var pkg = new Package(deploymentConfig, lock, expr.package, pkgName, "workspace:" + pkgPath, baseDir, production, expr.sourcesCache);
+                                    pkg.source.symlink = true; // Symlink workspace interdependencies
+                                    slasp.sequence([
+                                        function(callback) {
+                                            pkg.source.fetch(callback);
+                                        },
+                                        function(callback) {
+                                            expr.sourcesCache.addSource(pkg.source);
+                                            expr.package.providedDependencies[pkgName] = pkg;
+                                            pkg.resolveDependenciesAndSources(callback);
+                                        },
+                                    ], callback);
+                                }, callback);
+                            },
+                        ], callback);
+                    } else {
+                        callback();
+                    }
+                }
             } else {
                 callback("The provided JSON file must consist of an object or an array");
             }
         },
 
+        /* If a Yarn workspace was specified, add source overrides for workspace packages. */
+        function(callback) {
+            if(yarnWorkspaceJSON !== undefined) {
+                var config = JSON.parse(fs.readFileSync(yarnWorkspaceJSON));
+                slasp.sequence([
+                    function(callback) {
+                        var resolveDir = path.dirname(yarnWorkspaceJSON);
+                        resolveYarnWorkspaces(config.workspaces, baseDir, resolveDir, callback);
+                    },
+                    function(callback, pkgs) {
+                        for(var pkgName in pkgs) {
+                            var pkgPath = pkgs[pkgName];
+                            deploymentConfig.packageOverrides[pkgName] = new Package(deploymentConfig, lock, null, pkgName, 'workspace:' + pkgPath, baseDir, true /* Never include development dependencies of transitive dependencies */, expr.sourcesCache);
+                        }
+                        callback();
+                    }
+                ], callback);
+            } else {
+                callback();
+            }
+        },
+
+        function(callback) {
+            expr.resolveDependencies(callback);
+        },
+
         /* Write the output Nix expression to the specified output file */
         function(callback) {
             fs.writeFile(outputNix, disclaimer + nijs.jsToNix(expr, true), callback);

lib/sources/LocalSource.js

@@ -93,19 +93,24 @@ LocalSource.prototype.convertFromLockedDependency = function(dependencyObj, call
 };
 
 /**
- * @see NixASTNode#toNixAST
+ * Create a NixFile AST node for the package source path.
  */
-LocalSource.prototype.toNixAST = function() {
-    var ast = Source.prototype.toNixAST.call(this);
-
+LocalSource.prototype.fileExpression = function() {
     if(this.srcPath === "./") {
-        ast.src = new nijs.NixFile({ value: "./." }); // ./ is not valid in the Nix expression language
+        return new nijs.NixFile({ value: "./." }); // ./ is not valid in the Nix expression language
     } else if(this.srcPath === "..") {
-        ast.src = new nijs.NixFile({ value: "./.." }); // .. is not valid in the Nix expression language
+        return new nijs.NixFile({ value: "./.." }); // .. is not valid in the Nix expression language
     } else {
-        ast.src = new nijs.NixFile({ value: this.srcPath });
+        return new nijs.NixFile({ value: this.srcPath });
     }
+};
 
+/**
+ * @see NixASTNode#toNixAST
+ */
+LocalSource.prototype.toNixAST = function() {
+    var ast = Source.prototype.toNixAST.call(this);
+    ast.src = this.fileExpression();
     return ast;
 };
 

lib/sources/Source.js

@@ -43,6 +43,7 @@ Source.constructSource = function(registryURL, registryAuthToken, baseDir, outpu
     var GitSource = require('./GitSource.js').GitSource;
     var HTTPSource = require('./HTTPSource.js').HTTPSource;
     var LocalSource = require('./LocalSource.js').LocalSource;
+    var WorkspaceSource = require('./WorkspaceSource.js').WorkspaceSource;
     var NPMRegistrySource = require('./NPMRegistrySource.js').NPMRegistrySource;
 
     var parsedVersionSpec = semver.validRange(versionSpec, true);
@@ -66,6 +67,8 @@ Source.constructSource = function(registryURL, registryAuthToken, baseDir, outpu
         return new GitSource(baseDir, dependencyName, "git://github.com/"+versionSpec);
     } else if(parsedUrl.protocol == "file:") { // If the version is a file URL, simply compose a Nix path
         return new LocalSource(baseDir, dependencyName, outputDir, parsedUrl.path);
+    } else if(parsedUrl.protocol == "workspace:") { // If the version is a workspace URL, the package is provided as a Nix expression
+        return new WorkspaceSource(baseDir, dependencyName, outputDir, versionSpec.slice(10));
     } else if(versionSpec.substr(0, 3) == "../" || versionSpec.substr(0, 2) == "~/" || versionSpec.substr(0, 2) == "./" || versionSpec.substr(0, 1) == "/") { // If the version is a path, simply compose a Nix path
         return new LocalSource(baseDir, dependencyName, outputDir, versionSpec);
     } else { // In all other cases, just try the registry. Sometimes invalid semver ranges are encountered or a tag has been provided (e.g. 'latest', 'unstable')
@@ -85,6 +88,16 @@ Source.prototype.fetch = function(callback) {
     callback("fetch() is not implemented, please use a prototype that inherits from Source");
 };
 
+/**
+ * Whether the version of this source satisfies the given version specifier.
+ *
+ * @method
+ * @param {String} versionSpec Version specifier to commpare
+ */
+Source.prototype.versionSatisfies = function(versionSpec) {
+    return semver.satisfies(this.config.version, versionSpec, true);
+};
+
 /**
  * Takes a dependency object from a lock file and converts it into a source object.
  *
@@ -117,12 +130,21 @@ Source.prototype.convertIntegrityStringToNixHash = function(integrity) {
     }
 };
 
+/**
+ * Return a version of the package name suitable for use as a Nix variable.
+ *
+ * Escapes characters from scoped package names that aren't allowed.
+ */
+Source.prototype.variableName = function() {
+    return this.config.name.replace("@", "_at_").replace("/", "_slash_");
+};
+
 /**
  * @see NixASTNode#toNixAST
  */
 Source.prototype.toNixAST = function() {
     return {
-        name: this.config.name.replace("@", "_at_").replace("/", "_slash_"), // Escape characters from scoped package names that aren't allowed
+        name: this.variableName(),
         packageName: this.config.name,
         version: this.config.version
     };