diff --git a/link.go b/link.go index d44e6ef..0ef8c9d 100644 --- a/link.go +++ b/link.go @@ -19,30 +19,30 @@ var LinkCommand = cli.Command{ Usage: "Symlink packages to their dvcsimport repos, for local development.", Description: `gx-go link eases local development by symlinking actual workspace repositories on demand. -Example workflow: - -> gx-go link QmQA5mdxru8Bh6dpC9PJfSkumqnmHgJX7knxSgBo5Lpime QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52 -linked QmQA5mdxru8Bh6dpC9PJfSkumqnmHgJX7knxSgBo5Lpime /home/user/go/src/github.com/libp2p/go-libp2p -linked QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw /home/user/go/src/github.com/multiformats/go-multihash -linked QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52 /home/user/go/src/github.com/ipfs/go-log - -> gx-go link -QmQA5mdxru8Bh6dpC9PJfSkumqnmHgJX7knxSgBo5Lpime /home/user/go/src/github.com/libp2p/go-libp2p -QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52 /home/user/go/src/github.com/ipfs/go-log -QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw /home/user/go/src/github.com/multiformats/go-multihash +gx-go link replaces the target gx package (either by name or hash) with +a symlink to the appropriate dvcs repo in your GOPATH. To make this +"symlinked" repo usable as a gx package, gx-go link rewrites the target +package's dvcs imports using the target package's package.json. +Unfortunately, this can cause build errors in packages depending on this +package if these dependent packages specify alternative, conflicting +dependency versions. We can work around this using the --override-deps flag +to rewrite the target package using dependencies from the current package +(the package you're trying to build) first, falling back on the target +package's package.json file. -> gx-go link -r QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52 -unlinked QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52 /home/user/go/src/github.com/ipfs/go-log +Example workflow: -> gx-go link -QmQA5mdxru8Bh6dpC9PJfSkumqnmHgJX7knxSgBo5Lpime /home/user/go/src/github.com/libp2p/go-libp2p -QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw /home/user/go/src/github.com/multiformats/go-multihash +> cd $GOPATH/src/github.com/ipfs/go-ipfs +> gx-go link --override-deps go-unixfs +Replaced 39 entries in the rewrite map: + github.com/ipfs/go-ipfs-chunker + github.com/ipfs/go-ipfs-blockstore + github.com/libp2p/go-libp2p-net + [...] +linked go-unixfs /home/user/go/src/github.com/ipfs/go-unixfs > gx-go link -r -a -unlinked QmQA5mdxru8Bh6dpC9PJfSkumqnmHgJX7knxSgBo5Lpime /home/user/go/src/github.com/libp2p/go-libp2p -unlinked QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw /home/user/go/src/github.com/multiformats/go-multihash - -> gx-go link +unlinked go-unixfs /home/user/go/src/github.com/ipfs/go-unixfs `, Flags: []cli.Flag{ cli.BoolFlag{ @@ -53,13 +53,20 @@ unlinked QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw /home/user/go/src/github Name: "a,all", Usage: "Remove all existing symlinks and reinstate the gx packages. Use with -r.", }, + cli.BoolFlag{ + Name: "o,override-deps", + Usage: "Override dependency versions of the target package with its current dependant package.", + }, }, Action: func(c *cli.Context) error { remove := c.Bool("remove") all := c.Bool("all") + overrideDeps := c.Bool("override-deps") + + depRefs := c.Args()[:] + // It can either be a hash or a name. - hashes := c.Args()[:] - if len(hashes) == 0 { + if len(depRefs) == 0 { links, err := listLinkedPackages() if err != nil { return err @@ -67,7 +74,7 @@ unlinked QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw /home/user/go/src/github if remove && all { for _, link := range links { - hashes = append(hashes, link[0]) + depRefs = append(depRefs, link[0]) } } @@ -78,28 +85,37 @@ unlinked QmVGtdTZdTFaLsaj2RwdVG8jcjNNcp1DE914DKZ2kHmXHw /home/user/go/src/github return nil } } - pkg, _ := LoadPackageFile(gx.PkgFileName) - for _, hash := range hashes { - if pkg != nil { - // try to resolve - if dep := pkg.FindDep(hash); dep != nil { - hash = dep.Hash - } + // Path of the current dependant package. + parentPackagePath, err := gx.GetPackageRoot() + if err != nil { + return fmt.Errorf("error retrieving the parent package: %s", err) + } + + parentPkg, err := LoadPackageFile(filepath.Join(parentPackagePath, gx.PkgFileName)) + if err != nil { + return fmt.Errorf("parent package not found in %s: %s", + parentPackagePath, err) + } + + for _, ref := range depRefs { + dep := parentPkg.FindDep(ref) + if dep == nil { + return fmt.Errorf("dependency reference not found in the parent package: %s", ref) } if remove { - target, err := unlinkPackage(hash) + target, err := unlinkDependency(dep) if err != nil { return err } - fmt.Printf("unlinked %s %s\n", hash, target) + fmt.Printf("unlinked %s %s\n", dep.Name, target) } else { - target, err := linkPackage(hash) + target, err := linkDependency(dep, overrideDeps, parentPackagePath) if err != nil { return err } - fmt.Printf("linked %s %s\n", hash, target) + fmt.Printf("linked %s %s\n", dep.Name, target) } } @@ -141,38 +157,38 @@ func listLinkedPackages() ([][]string, error) { return links, nil } -// gx get $hash -// go get $dvcsimport -// rm -rf $GOPATH/src/gx/ipfs/$hash/$pkgname -// ln -s $GOPATH/src/$dvcsimport $GOPATH/src/gx/ipfs/$hash/$pkgname -// cd $GOPATH/src/$dvcsimport && gx install && gx-go rewrite -func linkPackage(hash string) (string, error) { - srcdir, err := gx.InstallPath("go", "", true) +// Link the dependency package `dep` to the global Gx workspace. +// +// The dependency is first fetched to find its DVCS import path (`gx get`), +// then the repository is fetched through `go get` and sym-linked: +// `$GOPATH/gx/ipfs///` -> `$GOPATH/src/` +// (`target`) -> (`linkPath`) +// If `overrideDeps` is set pass the option to the `post-install` hook to override +// dependency versions. +func linkDependency(dep *gx.Dependency, overrideDeps bool, parentPackagePath string) (string, error) { + gxSrcDir, err := gx.InstallPath("go", "", true) if err != nil { return "", err } - gxdir := filepath.Join(srcdir, "gx", "ipfs", hash) - - gxget := exec.Command("gx", "get", hash, "-o", gxdir) - gxget.Stdout = os.Stderr - gxget.Stderr = os.Stderr - if err = gxget.Run(); err != nil { - return "", fmt.Errorf("error during gx get: %s", err) - } - var pkg gx.Package - err = gx.FindPackageInDir(&pkg, gxdir) + dvcsImport, err := findDepDVCSimport(dep, gxSrcDir) if err != nil { - return "", fmt.Errorf("error during gx.FindPackageInDir: %s", err) + return "", fmt.Errorf("error trying to get the DVCS import" + + "of the dependeny %s: %s", dep.Name, err) } - dvcsimport := GxDvcsImport(&pkg) - target := filepath.Join(srcdir, dvcsimport) - gxtarget := filepath.Join(gxdir, pkg.Name) + target := filepath.Join(gxSrcDir, dvcsImport) + + // Linked package directory, needed for the `post-install` hook. + linkPackageDir := filepath.Join(gxSrcDir, "gx", "ipfs", dep.Hash) + // TODO: this shouldn't be necessary, we should be able to just pass the + // `linkPath` (i.e., the directory with the name of the package). + + linkPath := filepath.Join(linkPackageDir, dep.Name) _, err = os.Stat(target) if os.IsNotExist(err) { - goget := exec.Command("go", "get", dvcsimport+"/...") + goget := exec.Command("go", "get", dvcsImport+"/...") goget.Stdout = nil goget.Stderr = os.Stderr if err = goget.Run(); err != nil { @@ -182,12 +198,12 @@ func linkPackage(hash string) (string, error) { return "", fmt.Errorf("error during os.Stat: %s", err) } - err = os.RemoveAll(gxtarget) + err = os.RemoveAll(linkPath) if err != nil { return "", fmt.Errorf("error during os.RemoveAll: %s", err) } - err = os.Symlink(target, gxtarget) + err = os.Symlink(target, linkPath) if err != nil { return "", fmt.Errorf("error during os.Symlink: %s", err) } @@ -200,53 +216,75 @@ func linkPackage(hash string) (string, error) { return "", fmt.Errorf("error during gx install: %s", err) } - rwcmd := exec.Command("gx-go", "hook", "post-install", gxdir) + rwcmdArgs := []string{"hook", "post-install", linkPackageDir} + if overrideDeps { + rwcmdArgs = append(rwcmdArgs, "--override-deps", parentPackagePath) + } + rwcmd := exec.Command("gx-go", rwcmdArgs...) rwcmd.Dir = target rwcmd.Stdout = os.Stdout rwcmd.Stderr = os.Stderr if err := rwcmd.Run(); err != nil { return "", fmt.Errorf("error during gx-go rw: %s", err) } + // TODO: Wrap command calls in a function. return target, nil } -// rm -rf $GOPATH/src/gx/ipfs/$hash -// gx get $hash -func unlinkPackage(hash string) (string, error) { - srcdir, err := gx.InstallPath("go", "", true) +// Return the DVCS import path of a dependency (fetching it +// if necessary). +func findDepDVCSimport(dep *gx.Dependency, gxSrcDir string) (string, error) { + gxdir := filepath.Join(gxSrcDir, "gx", "ipfs", dep.Hash) + + // Get the dependency to find out its DVCS import. + err := gxGetPackage(dep.Hash) if err != nil { return "", err } - gxdir := filepath.Join(srcdir, "gx", "ipfs", hash) - err = os.RemoveAll(gxdir) + var pkg gx.Package + err = gx.FindPackageInDir(&pkg, gxdir) if err != nil { - return "", fmt.Errorf("error during os.RemoveAll: %s", err) + return "", fmt.Errorf("error during gx.FindPackageInDir: %s", err) } - gxget := exec.Command("gx", "get", hash, "-o", gxdir) - gxget.Stdout = nil - gxget.Stderr = os.Stderr - if err = gxget.Run(); err != nil { - return "", fmt.Errorf("error during gx get: %s", err) + return GxDvcsImport(&pkg), nil +} + +// rm -rf $GOPATH/src/gx/ipfs/$hash +// gx get $hash +func unlinkDependency(dep *gx.Dependency) (string, error) { + gxSrcDir, err := gx.InstallPath("go", "", true) + if err != nil { + return "", err } - var pkg gx.Package - err = gx.FindPackageInDir(&pkg, gxdir) + dvcsImport, err := findDepDVCSimport(dep, gxSrcDir) if err != nil { - return "", fmt.Errorf("error during gx.FindPackageInDir: %s", err) + return "", fmt.Errorf("error trying to get the DVCS import of the dependeny %s: %s", dep.Name, err) } - dvcsimport := GxDvcsImport(&pkg) - target := filepath.Join(srcdir, dvcsimport) + target := filepath.Join(gxSrcDir, dvcsImport) - uwcmd := exec.Command("gx-go", "uw") + uwcmd := exec.Command("gx-go", "rw", "--fix") + // The `--fix` options is more time consuming (compared to the normal + // `gx-go uw` call) but as some of the import paths may have been written + // from synced dependencies (`gx-go link --sync`) of another package that + // may not be available now (to build the rewrite map) this is the safer + // option. uwcmd.Dir = target uwcmd.Stdout = nil uwcmd.Stderr = os.Stderr if err := uwcmd.Run(); err != nil { - return "", fmt.Errorf("error during gx-go uw: %s", err) + return "", fmt.Errorf("error during gx-go rw: %s", err) + } + + // Remove the package at the end as `gx-go rw --fix` will need to use it + // (to find the DVCS import paths). + err = os.RemoveAll(filepath.Join(gxSrcDir, "gx", "ipfs", dep.Hash)) + if err != nil { + return "", fmt.Errorf("error during os.RemoveAll: %s", err) } return target, nil diff --git a/main.go b/main.go index 9f00bd1..b7cb0bb 100644 --- a/main.go +++ b/main.go @@ -392,6 +392,23 @@ func goGetPackage(path string) error { return nil } +func gxGetPackage(hash string) error { + srcdir, err := gx.InstallPath("go", "", true) + if err != nil { + return err + } + gxdir := filepath.Join(srcdir, "gx", "ipfs", hash) + + gxget := exec.Command("gx", "get", hash, "-o", gxdir) + gxget.Stdout = os.Stderr + gxget.Stderr = os.Stderr + if err = gxget.Run(); err != nil { + return fmt.Errorf("error during gx get: %s", err) + } + + return nil +} + func fixImports(path string) error { fixmap := make(map[string]string) gopath := os.Getenv("GOPATH") @@ -411,9 +428,19 @@ func fixImports(path string) error { var pkg Package err := gx.FindPackageInDir(&pkg, filepath.Join(gopath, "src", canon)) if err != nil { - fmt.Println(err) - return imp + hash := parts[2] + err = gxGetPackage(hash) + if err != nil { + VLog(err) + return imp + } + err := gx.FindPackageInDir(&pkg, filepath.Join(gopath, "src", canon)) + if err != nil { + VLog(err) + return imp + } } + if pkg.Gx.DvcsImport != "" { fixmap[imp] = pkg.Gx.DvcsImport return pkg.Gx.DvcsImport + rest @@ -592,6 +619,10 @@ var postInstallHookCommand = cli.Command{ Name: "global", Usage: "specifies whether or not the install was global", }, + cli.StringFlag{ + Name: "override-deps", + Usage: "path of a package used to override dependency versions (intended to be used with the link command)", + }, }, Action: func(c *cli.Context) error { if !c.Args().Present() { @@ -621,10 +652,57 @@ var postInstallHookCommand = cli.Command{ reldir = dir } + var depsPkg *Package + var depsPkgDir string + if depsPkgDir = c.String("override-deps"); depsPkgDir != "" { + err := gx.FindPackageInDir(&depsPkg, depsPkgDir) + if err != nil { + return fmt.Errorf("find deps package in %s failed : %s", depsPkgDir, err) + } + } + mapping := make(map[string]string) err = buildRewriteMapping(&pkg, reldir, mapping, false) if err != nil { - return fmt.Errorf("building rewrite mapping failed: %s", err) + return fmt.Errorf("building rewrite mapping failed for package %s: %s", pkg.Name, err) + } + + if depsPkg != nil { + // Use the dependency versions of `depsPkg` in case of a mismatch + // with the versions in `pkg`. + + depsRewriteMap := make(map[string]string) + err = buildRewriteMapping(depsPkg, depsPkgDir, depsRewriteMap, false) + if err != nil { + return fmt.Errorf("building rewrite mapping failed for package %s: %s", depsPkg.Name, err) + // TODO: All the dependencies of the deps package need to be fetched. Should we call + // `gx install --global`? + } + + // Iterate the `rewriteMap` indexed by the DVCS imports (since `undo` + // is false) and replace them with dependencies found in the + // `depsRewriteMap` if the gx import paths don't match (that is, + // if their versions are different). + var replacedImports []string + for dvcsImport, gxImportPath := range mapping { + depsGxImportPath, exists := depsRewriteMap[dvcsImport] + + if exists && gxImportPath != depsGxImportPath { + mapping[dvcsImport] = depsGxImportPath + replacedImports = append(replacedImports, dvcsImport) + } + } + + if len(replacedImports) > 0 { + fmt.Printf("Replaced %d entries in the rewrite map:\n", len(replacedImports)) + for _, dvcsImport := range(replacedImports) { + fmt.Printf(" %s\n", dvcsImport) + } + } + // TODO: This should be handled by the `VLog` function. + // (At the moment this is called from `gx-go link` which doesn't + // have access to the global `Verbose` flag to check whether to + // include the `--verbose` argument or not.) } hash := filepath.Base(npkg) @@ -987,22 +1065,44 @@ func globalPath() string { return filepath.Join(gp, "src", "gx", "ipfs") } -func loadDep(dep *gx.Dependency, pkgdir string) (*Package, error) { - var cpkg Package - pdir := filepath.Join(pkgdir, dep.Hash) - VLog(" - fetching dep: %s (%s)", dep.Name, dep.Hash) - err := gx.FindPackageInDir(&cpkg, pdir) +// Load the `Dependency` by its hash returning the `Package` where it's +// installed, `pkgDir` is an optional parameter with the directory +// where to look for that dependency. +// TODO: `pkgDir` isn't actually the package directory, it's where +// *all* the packages are stored, it should have another name (and +// it shouldn't be "packages directory"). +func loadDep(dep *gx.Dependency, pkgDir string) (*Package, error) { + var pkg Package + if pkgDir != "" { + pkgPath := filepath.Join(pkgDir, dep.Hash) + VLog(" - fetching dep: %s (%s)", dep.Name, dep.Hash) + err := gx.FindPackageInDir(&pkg, pkgPath) + if err == nil { + return &pkg, nil + } + } + + // Either `pkgDir` wasn't specified or it failed + // to find it there, try global path. + p := filepath.Join(globalPath(), dep.Hash) + VLog(" - checking in global namespace (%s)", p) + err := gx.FindPackageInDir(&pkg, p) if err != nil { - // try global - p := filepath.Join(globalPath(), dep.Hash) - VLog(" - checking in global namespace (%s)", p) - gerr := gx.FindPackageInDir(&cpkg, p) - if gerr != nil { - return nil, fmt.Errorf("failed to find package: %s", gerr) + // It didn't find the package in the glogal path, try + // fetching it. + err := gxGetPackage(dep.Hash) + // TODO: This works because `gxGetPackage` has the global path hard-coded. + if err != nil { + return nil, fmt.Errorf("failed to fetch package: %s", err) + } + + err = gx.FindPackageInDir(&pkg, p) + if err != nil { + return nil, fmt.Errorf("failed to find package: %s", err) } } - return &cpkg, nil + return &pkg, nil } // Rewrites the package `DvcsImport` with the dependency hash (or @@ -1030,6 +1130,9 @@ func addRewriteForDep(dep *gx.Dependency, pkg *Package, m map[string]string, und } func buildRewriteMapping(pkg *Package, pkgdir string, m map[string]string, undo bool) error { + // TODO: Encapsulate `Package` and `pkgDir` in another structure + // (such as `installedPackage`). + seen := make(map[string]struct{}) var process func(pkg *Package, rootPackage bool) error