From ef18aeaae22197e72f553192613b9d8edb1ef9c6 Mon Sep 17 00:00:00 2001 From: LiusCraft Date: Fri, 23 Feb 2024 21:54:07 +0800 Subject: [PATCH 1/4] feat: go list all pkgs and cache them to improve build speed --- packages/imp.go | 61 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/packages/imp.go b/packages/imp.go index a7e4e9e7..66646aa3 100644 --- a/packages/imp.go +++ b/packages/imp.go @@ -20,11 +20,21 @@ import ( "go/types" "os" "os/exec" + "strings" "sync" "golang.org/x/tools/go/gcexportdata" ) +// pkgPath Caches +var ( + dirCache = map[string]bool{} + dirCacheMutex = sync.RWMutex{} + packageCacheMap = map[string]string{} + packageCacheMutex = sync.RWMutex{} + waitCache = sync.WaitGroup{} +) + // ---------------------------------------------------------------------------- type Importer struct { @@ -40,6 +50,9 @@ func NewImporter(fset *token.FileSet, workDir ...string) *Importer { if len(workDir) > 0 { dir = workDir[0] } + if len(packageCacheMap) == 0 { + initGoListCache(dir) + } if fset == nil { fset = token.NewFileSet() } @@ -96,6 +109,11 @@ func (p *Importer) loadByExport(expfile string, pkgPath string) (ret *types.Pack // FindExport lookups export file (.a) of a package by its pkgPath. // It returns empty if pkgPath not found. func FindExport(dir, pkgPath string) (expfile string, err error) { + waitCache.Wait() + expfile = packageCacheMap[pkgPath] + if len(expfile) > 0 { + return expfile, nil + } data, err := golistExport(dir, pkgPath) if err != nil { return @@ -104,6 +122,49 @@ func FindExport(dir, pkgPath string) (expfile string, err error) { return } +// https://github.com/goplus/gop/issues/1710 +// Not fully optimized +// Retrieve all imports in the specified directory and cache them +func goListExportCache(dir string, pkgs ...string) { + dirCacheMutex.Lock() + if dirCache[dir] { + dirCacheMutex.Unlock() + return + } + dirCache[dir] = true + dirCacheMutex.Unlock() + var stdout, stderr bytes.Buffer + commandStr := []string{"list", "-f", "{{.ImportPath}},{{.Export}}", "-export", "-e"} + commandStr = append(commandStr, pkgs...) + commandStr = append(commandStr, "all") + cmd := exec.Command("go", commandStr...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.Dir = dir + err := cmd.Run() + if err == nil { + ret := stdout.String() + for _, v := range strings.Split(ret, "\n") { + s := strings.Split(v, ",") + if len(s) != 2 { + continue + } + packageCacheMutex.Lock() + packageCacheMap[s[0]] = s[1] + packageCacheMutex.Unlock() + } + } +} +func GoListExportCacheSync(dir string, pkgs ...string) { + waitCache.Add(1) + go func() { + defer waitCache.Done() + goListExportCache(dir, pkgs...) + }() +} +func initGoListCache(dir string) { + goListExportCache(dir) +} func golistExport(dir, pkgPath string) (ret []byte, err error) { var stdout, stderr bytes.Buffer cmd := exec.Command("go", "list", "-f={{.Export}}", "-export", pkgPath) From cf873f64c2860f6878d242b5425757cda8da0fb8 Mon Sep 17 00:00:00 2001 From: LiusCraft Date: Mon, 26 Feb 2024 18:06:02 +0800 Subject: [PATCH 2/4] test: add test case: packages.GoListExportCacheSync --- packages/imp_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/imp_test.go b/packages/imp_test.go index 1fe69ad8..02ac2740 100644 --- a/packages/imp_test.go +++ b/packages/imp_test.go @@ -56,4 +56,13 @@ func Test_loadByExport(t *testing.T) { FindExport(".", "C") } +func TestGoListExportCacheSync(t *testing.T) { + GoListExportCacheSync("..") + p := NewImporter(nil, "..") + pkg, err := p.Import("github.com/goplus/gox/internal/foo") + if err != nil { + t.Fatal("Import failed:", pkg, err) + } +} + // ---------------------------------------------------------------------------- From dbec0f67e51031152047de493146f0b1474fbe35 Mon Sep 17 00:00:00 2001 From: LiusCraft Date: Tue, 19 Mar 2024 11:29:09 +0800 Subject: [PATCH 3/4] fix: the go+ code will not be updated after the dependent package exists in autogen --- packages/imp.go | 80 +++++++++---------------------------------------- 1 file changed, 14 insertions(+), 66 deletions(-) diff --git a/packages/imp.go b/packages/imp.go index 66646aa3..17860baa 100644 --- a/packages/imp.go +++ b/packages/imp.go @@ -20,28 +20,19 @@ import ( "go/types" "os" "os/exec" - "strings" "sync" "golang.org/x/tools/go/gcexportdata" ) -// pkgPath Caches -var ( - dirCache = map[string]bool{} - dirCacheMutex = sync.RWMutex{} - packageCacheMap = map[string]string{} - packageCacheMutex = sync.RWMutex{} - waitCache = sync.WaitGroup{} -) - // ---------------------------------------------------------------------------- type Importer struct { - loaded map[string]*types.Package - fset *token.FileSet - dir string - m sync.RWMutex + loaded map[string]*types.Package + fset *token.FileSet + dir string + m sync.RWMutex + pkgCache *Cache } // NewImporter creates an Importer object that meets types.Importer interface. @@ -50,15 +41,12 @@ func NewImporter(fset *token.FileSet, workDir ...string) *Importer { if len(workDir) > 0 { dir = workDir[0] } - if len(packageCacheMap) == 0 { - initGoListCache(dir) - } if fset == nil { fset = token.NewFileSet() } loaded := make(map[string]*types.Package) loaded["unsafe"] = types.Unsafe - return &Importer{loaded: loaded, fset: fset, dir: dir} + return &Importer{loaded: loaded, fset: fset, dir: dir, pkgCache: NewGoListCache(dir)} } func (p *Importer) Import(pkgPath string) (pkg *types.Package, err error) { @@ -81,6 +69,10 @@ func (p *Importer) ImportFrom(pkgPath, dir string, mode types.ImportMode) (*type return ret, nil } p.m.RUnlock() + cacheInfo, ok := p.pkgCache.GetPkgCache(pkgPath) + if ok { + return p.loadByExport(cacheInfo.PkgExport, pkgPath) + } expfile, err := FindExport(dir, pkgPath) if err != nil { return nil, err @@ -104,16 +96,15 @@ func (p *Importer) loadByExport(expfile string, pkgPath string) (ret *types.Pack return } +func (p *Importer) GetPkgCache() *Cache { + return p.pkgCache +} + // ---------------------------------------------------------------------------- // FindExport lookups export file (.a) of a package by its pkgPath. // It returns empty if pkgPath not found. func FindExport(dir, pkgPath string) (expfile string, err error) { - waitCache.Wait() - expfile = packageCacheMap[pkgPath] - if len(expfile) > 0 { - return expfile, nil - } data, err := golistExport(dir, pkgPath) if err != nil { return @@ -122,49 +113,6 @@ func FindExport(dir, pkgPath string) (expfile string, err error) { return } -// https://github.com/goplus/gop/issues/1710 -// Not fully optimized -// Retrieve all imports in the specified directory and cache them -func goListExportCache(dir string, pkgs ...string) { - dirCacheMutex.Lock() - if dirCache[dir] { - dirCacheMutex.Unlock() - return - } - dirCache[dir] = true - dirCacheMutex.Unlock() - var stdout, stderr bytes.Buffer - commandStr := []string{"list", "-f", "{{.ImportPath}},{{.Export}}", "-export", "-e"} - commandStr = append(commandStr, pkgs...) - commandStr = append(commandStr, "all") - cmd := exec.Command("go", commandStr...) - cmd.Stdout = &stdout - cmd.Stderr = &stderr - cmd.Dir = dir - err := cmd.Run() - if err == nil { - ret := stdout.String() - for _, v := range strings.Split(ret, "\n") { - s := strings.Split(v, ",") - if len(s) != 2 { - continue - } - packageCacheMutex.Lock() - packageCacheMap[s[0]] = s[1] - packageCacheMutex.Unlock() - } - } -} -func GoListExportCacheSync(dir string, pkgs ...string) { - waitCache.Add(1) - go func() { - defer waitCache.Done() - goListExportCache(dir, pkgs...) - }() -} -func initGoListCache(dir string) { - goListExportCache(dir) -} func golistExport(dir, pkgPath string) (ret []byte, err error) { var stdout, stderr bytes.Buffer cmd := exec.Command("go", "list", "-f={{.Export}}", "-export", pkgPath) From aced8c57161feded6c5a02ad07a7479f5620d749 Mon Sep 17 00:00:00 2001 From: LiusCraft Date: Tue, 19 Mar 2024 17:59:51 +0800 Subject: [PATCH 4/4] test: add test cases for pkgCache --- packages/cache.go | 95 ++++++++++++++++++++++++++++++++++++++++++ packages/cache_test.go | 18 ++++++++ packages/imp_test.go | 9 ---- 3 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 packages/cache.go create mode 100644 packages/cache_test.go diff --git a/packages/cache.go b/packages/cache.go new file mode 100644 index 00000000..5cbe2a03 --- /dev/null +++ b/packages/cache.go @@ -0,0 +1,95 @@ +package packages + +import ( + "bytes" + "os/exec" + "strings" + "sync" +) + +// pkgPath Caches +type Cache struct { + dirCache map[string]bool + dirCacheMutex sync.RWMutex + packageCacheMap map[string]CacheInfo + packageCacheMutex sync.RWMutex + waitCache sync.WaitGroup +} + +type CacheInfo struct { + ImportPath string + PkgDir string + PkgExport string +} + +// https://github.com/goplus/gop/issues/1710 +// Not fully optimized +// Retrieve all imports in the specified directory and cache them +func (c *Cache) goListExportCache(dir string, pkgs ...string) { + c.dirCacheMutex.Lock() + if c.dirCache[dir] { + c.dirCacheMutex.Unlock() + return + } + c.dirCache[dir] = true + c.dirCacheMutex.Unlock() + var stdout, stderr bytes.Buffer + commandStr := []string{"list", "-f", "{{.ImportPath}},{{.Dir}},{{.Export}}", "-export", "-e"} + commandStr = append(commandStr, pkgs...) + commandStr = append(commandStr, "all") + cmd := exec.Command("go", commandStr...) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.Dir = dir + err := cmd.Run() + if err == nil { + ret := stdout.String() + for _, v := range strings.Split(ret, "\n") { + s := strings.Split(v, ",") + if len(s) != 3 { + continue + } + c.packageCacheMutex.Lock() + c.packageCacheMap[s[0]] = CacheInfo{s[0], s[1], s[2]} + c.packageCacheMutex.Unlock() + } + } +} + +// Reduce wait time when performing `go list` on multiple pkgDir at the same time +func (c *Cache) GoListExportCacheSync(dir string, pkgs ...string) { + c.waitCache.Add(1) + go func() { + defer c.waitCache.Done() + c.goListExportCache(dir, pkgs...) + }() +} + +func (c *Cache) GetPkgCache(pkgPath string) (ret CacheInfo, ok bool) { + c.waitCache.Wait() + c.packageCacheMutex.RLock() + ret, ok = c.packageCacheMap[pkgPath] + c.packageCacheMutex.RUnlock() + return +} + +func (c *Cache) DelPkgCache(pkgPath string) bool { + _, ok := c.GetPkgCache(pkgPath) + if ok { + c.packageCacheMutex.Lock() + delete(c.packageCacheMap, pkgPath) + c.packageCacheMutex.Unlock() + return true + } + return false +} + +func NewGoListCache(dir string) *Cache { + c := &Cache{ + dirCache: make(map[string]bool), + packageCacheMap: make(map[string]CacheInfo), + } + // get the dir pkg cache + c.GoListExportCacheSync(dir) + return c +} diff --git a/packages/cache_test.go b/packages/cache_test.go new file mode 100644 index 00000000..ceef27fc --- /dev/null +++ b/packages/cache_test.go @@ -0,0 +1,18 @@ +package packages + +import "testing" + +func TestCacheDelCache(t *testing.T) { + p := NewImporter(nil) + if !p.GetPkgCache().DelPkgCache("fmt") { + t.Fatal("del cache should pass!") + } + if p.GetPkgCache().DelPkgCache("fmt") { + t.Fatal("del cache should fail") + } +} + +func TestRepeatLoadDir(t *testing.T) { + p := NewImporter(nil) + p.GetPkgCache().goListExportCache("", "") +} diff --git a/packages/imp_test.go b/packages/imp_test.go index 02ac2740..1fe69ad8 100644 --- a/packages/imp_test.go +++ b/packages/imp_test.go @@ -56,13 +56,4 @@ func Test_loadByExport(t *testing.T) { FindExport(".", "C") } -func TestGoListExportCacheSync(t *testing.T) { - GoListExportCacheSync("..") - p := NewImporter(nil, "..") - pkg, err := p.Import("github.com/goplus/gox/internal/foo") - if err != nil { - t.Fatal("Import failed:", pkg, err) - } -} - // ----------------------------------------------------------------------------