From c2773f3fc65b4005cbea64631b7ca177896c7bdb Mon Sep 17 00:00:00 2001 From: Grant Nelson Date: Thu, 5 Dec 2024 10:51:05 -0700 Subject: [PATCH] Serializing Archieves --- build/build.go | 9 +-- build/cache/cache.go | 18 ++++-- build/cache/cache_test.go | 54 ++++++++++++++--- compiler/compiler.go | 40 +++++++------ compiler/compiler_test.go | 120 +++++++++++++++++++++----------------- compiler/package.go | 2 - 6 files changed, 150 insertions(+), 93 deletions(-) diff --git a/build/build.go b/build/build.go index 5748fae98..aea22d7b9 100644 --- a/build/build.go +++ b/build/build.go @@ -977,11 +977,8 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { } if !s.options.NoCache { - archive := s.buildCache.LoadArchive(pkg.ImportPath, s.Types) - if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) { - if err := archive.RegisterTypes(s.Types); err != nil { - panic(fmt.Errorf("failed to load type information from %v: %w", archive, err)) - } + archive := s.buildCache.LoadArchive(pkg.ImportPath, pkg.SrcModTime, s.Types) + if archive != nil { s.UpToDateArchives[pkg.ImportPath] = archive // Existing archive is up to date, no need to build it from scratch. return archive, nil @@ -1021,7 +1018,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) { fmt.Println(pkg.ImportPath) } - s.buildCache.StoreArchive(archive) + s.buildCache.StoreArchive(archive, time.Now()) s.UpToDateArchives[pkg.ImportPath] = archive return archive, nil diff --git a/build/cache/cache.go b/build/cache/cache.go index 0d7f99b33..90473c7fe 100644 --- a/build/cache/cache.go +++ b/build/cache/cache.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "time" "github.com/gopherjs/gopherjs/compiler" log "github.com/sirupsen/logrus" @@ -91,7 +92,7 @@ func (bc BuildCache) String() string { // StoreArchive compiled archive in the cache. Any error inside this method // will cause the cache not to be persisted. -func (bc *BuildCache) StoreArchive(a *compiler.Archive) { +func (bc *BuildCache) StoreArchive(a *compiler.Archive, buildTime time.Time) { if bc == nil { return // Caching is disabled. } @@ -107,7 +108,7 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) { return } defer f.Close() - if err := compiler.WriteArchive(a, f); err != nil { + if err := compiler.WriteArchive(a, buildTime, f); err != nil { log.Warningf("Failed to write build cache archive %q: %v", a, err) // Make sure we don't leave a half-written archive behind. os.Remove(f.Name()) @@ -126,7 +127,10 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) { // // The returned archive would have been built with the same configuration as // the build cache was. -func (bc *BuildCache) LoadArchive(importPath string, packages map[string]*types.Package) *compiler.Archive { +// +// The imports map is used to resolve package dependencies and may modify the +// map to include the package from the read archive. See [gcexportdata.Read]. +func (bc *BuildCache) LoadArchive(importPath string, srcModTime time.Time, imports map[string]*types.Package) *compiler.Archive { if bc == nil { return nil // Caching is disabled. } @@ -141,12 +145,16 @@ func (bc *BuildCache) LoadArchive(importPath string, packages map[string]*types. return nil // Cache miss. } defer f.Close() - a, err := compiler.ReadArchive(importPath, f, packages) + a, buildTime, err := compiler.ReadArchive(importPath, f, srcModTime, imports) if err != nil { log.Warningf("Failed to read cached package archive for %q: %v", importPath, err) return nil // Invalid/corrupted archive, cache miss. } - log.Infof("Found cached package archive for %q, built at %v.", importPath, a.BuildTime) + if a == nil { + log.Infof("Found out-of-date package archive for %q, built at %v.", importPath, buildTime) + return nil // Archive is out-of-date, cache miss. + } + log.Infof("Found cached package archive for %q, built at %v.", importPath, buildTime) return a } diff --git a/build/cache/cache_test.go b/build/cache/cache_test.go index e5179085d..0a0541f64 100644 --- a/build/cache/cache_test.go +++ b/build/cache/cache_test.go @@ -3,6 +3,7 @@ package cache import ( "go/types" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/gopherjs/gopherjs/compiler" @@ -16,22 +17,24 @@ func TestStore(t *testing.T) { Imports: []string{"fake/dep"}, } - deps := map[string]*types.Package{} + srcModTime := newTime(0.0) + buildTime := newTime(5.0) + imports := map[string]*types.Package{} bc := BuildCache{} - if got := bc.LoadArchive(want.ImportPath, deps); got != nil { + if got := bc.LoadArchive(want.ImportPath, srcModTime, imports); got != nil { t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath) } - bc.StoreArchive(want) - got := bc.LoadArchive(want.ImportPath, deps) + bc.StoreArchive(want, buildTime) + got := bc.LoadArchive(want.ImportPath, srcModTime, imports) if got == nil { - t.Errorf("Got: %s wan not found in the cache. Want: archive is can be loaded after store.", want.ImportPath) + t.Errorf("Got: %s was not found in the cache. Want: archive is can be loaded after store.", want.ImportPath) } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("Loaded archive is different from stored (-want,+got):\n%s", diff) } // Make sure the package names are a part of the cache key. - if got := bc.LoadArchive("fake/other", deps); got != nil { + if got := bc.LoadArchive("fake/other", srcModTime, imports); got != nil { t.Errorf("Got: fake/other was found in cache: %#v. Want: nil for packages that weren't cached.", got) } } @@ -61,21 +64,54 @@ func TestInvalidation(t *testing.T) { }, } - deps := map[string]*types.Package{} + srcModTime := newTime(0.0) + buildTime := newTime(5.0) + imports := map[string]*types.Package{} for _, test := range tests { a := &compiler.Archive{ImportPath: "package/fake"} - test.cache1.StoreArchive(a) + test.cache1.StoreArchive(a, buildTime) - if got := test.cache2.LoadArchive(a.ImportPath, deps); got != nil { + if got := test.cache2.LoadArchive(a.ImportPath, srcModTime, imports); got != nil { t.Logf("-cache1,+cache2:\n%s", cmp.Diff(test.cache1, test.cache2)) t.Errorf("Got: %v loaded from cache. Want: build parameter change invalidates cache.", got) } } } +func TestOldArchive(t *testing.T) { + cacheForTest(t) + + want := &compiler.Archive{ + ImportPath: "fake/package", + Imports: []string{"fake/dep"}, + } + + buildTime := newTime(5.0) + imports := map[string]*types.Package{} + bc := BuildCache{} + bc.StoreArchive(want, buildTime) + + oldSrcModTime := newTime(2.0) // older than archive build time, so archive is up-to-date + got := bc.LoadArchive(want.ImportPath, oldSrcModTime, imports) + if got == nil { + t.Errorf("Got: %s was nil. Want: up-to-date archive to be loaded.", want.ImportPath) + } + + newerSrcModTime := newTime(7.0) // newer than archive build time, so archive is stale + got = bc.LoadArchive(want.ImportPath, newerSrcModTime, imports) + if got != nil { + t.Errorf("Got: %s was not nil. Want: stale archive to not be loaded.", want.ImportPath) + } +} + func cacheForTest(t *testing.T) { t.Helper() originalRoot := cacheRoot t.Cleanup(func() { cacheRoot = originalRoot }) cacheRoot = t.TempDir() } + +func newTime(seconds float64) time.Time { + return time.Date(1969, 7, 20, 20, 17, 0, 0, time.UTC). + Add(time.Duration(seconds * float64(time.Second))) +} diff --git a/compiler/compiler.go b/compiler/compiler.go index 1fb8b83b9..f6f470ecd 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -60,20 +60,12 @@ type Archive struct { Minified bool // A list of go:linkname directives encountered in the package. GoLinknames []GoLinkname - // Time when this archive was built. - BuildTime time.Time } func (a Archive) String() string { return fmt.Sprintf("compiler.Archive{%s}", a.ImportPath) } -// RegisterTypes adds package type information from the archive into the provided map. -func (a *Archive) RegisterTypes(packages map[string]*types.Package) error { - packages[a.ImportPath] = a.Package - return nil -} - type Dependency struct { Pkg string Type string @@ -277,18 +269,30 @@ type serializableArchive struct { } // ReadArchive reads serialized compiled archive of the importPath package. -func ReadArchive(path string, r io.Reader, packages map[string]*types.Package) (*Archive, error) { +// +// The given srcModTime is used to determine if the archive is out-of-date. +// If the archive is out-of-date, the returned archive is nil. +// If there was not an error, the returned time is when the archive was built. +// +// The imports map is used to resolve package dependencies and may modify the +// map to include the package from the read archive. See [gcexportdata.Read]. +func ReadArchive(importPath string, r io.Reader, srcModTime time.Time, imports map[string]*types.Package) (*Archive, time.Time, error) { var sa serializableArchive if err := gob.NewDecoder(r).Decode(&sa); err != nil { - return nil, err + return nil, time.Time{}, err + } + + if srcModTime.After(sa.BuildTime) { + // Archive is out-of-date. + return nil, sa.BuildTime, nil } var a Archive fset := token.NewFileSet() if len(sa.ExportData) > 0 { - pkg, err := gcexportdata.Read(bytes.NewReader(sa.ExportData), fset, packages, a.ImportPath) + pkg, err := gcexportdata.Read(bytes.NewReader(sa.ExportData), fset, imports, importPath) if err != nil { - return nil, err + return nil, sa.BuildTime, err } a.Package = pkg } @@ -296,7 +300,7 @@ func ReadArchive(path string, r io.Reader, packages map[string]*types.Package) ( if len(sa.FileSet) > 0 { a.FileSet = token.NewFileSet() if err := a.FileSet.Read(json.NewDecoder(bytes.NewReader(sa.FileSet)).Decode); err != nil { - return nil, err + return nil, sa.BuildTime, err } } @@ -307,12 +311,14 @@ func ReadArchive(path string, r io.Reader, packages map[string]*types.Package) ( a.IncJSCode = sa.IncJSCode a.Minified = sa.Minified a.GoLinknames = sa.GoLinknames - a.BuildTime = sa.BuildTime - return &a, nil + return &a, sa.BuildTime, nil } // WriteArchive writes compiled package archive on disk for later reuse. -func WriteArchive(a *Archive, w io.Writer) error { +// +// The passed in buildTime is used to determine if the archive is out-of-date. +// It should be set to time.Now() typically but it exposed for testing purposes. +func WriteArchive(a *Archive, buildTime time.Time, w io.Writer) error { exportData := new(bytes.Buffer) if a.Package != nil { if err := gcexportdata.Write(exportData, nil, a.Package); err != nil { @@ -337,7 +343,7 @@ func WriteArchive(a *Archive, w io.Writer) error { FileSet: encodedFileSet.Bytes(), Minified: a.Minified, GoLinknames: a.GoLinknames, - BuildTime: a.BuildTime, + BuildTime: buildTime, } return gob.NewEncoder(w).Encode(sa) diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index 9d2d58de8..22a8f8ff8 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -6,6 +6,7 @@ import ( "regexp" "sort" "testing" + "time" "github.com/google/go-cmp/cmp" "golang.org/x/tools/go/packages" @@ -420,61 +421,16 @@ func TestArchiveSelectionAfterSerialization(t *testing.T) { }` srcFiles := []srctesting.Source{{Name: `main.go`, Contents: []byte(src)}} root := srctesting.ParseSources(t, srcFiles, nil) - archives := compileProject(t, root, false) + rootPath := root.PkgPath + origArchives := compileProject(t, root, false) + readArchives := reloadCompiledProject(t, origArchives, rootPath) - paths := make([]string, 0, len(archives)) - for path := range archives { - paths = append(paths, path) - } - sort.Strings(paths) - packages := make([]*Archive, 0, len(archives)) - for _, path := range paths { - // Serialize the archive - buf := bytes.Buffer{} - if err := WriteArchive(archives[path], &buf); err != nil { - t.Fatalf(`failed to write archive for %s: %v`, path, err) - } - - // Deserialize the archive to simulate loading from disk - deps := map[string]*types.Package{} - archive, err := ReadArchive(path, &buf, deps) - if err != nil { - t.Fatalf(`failed to read archive for %s: %v`, path, err) - } - - archives[path] = archive - packages = append(packages, archive) - } - - selector := &dce.Selector[*Decl]{} - for _, pkg := range packages { - for _, d := range pkg.Declarations { - selector.Include(d, false) - } - } - dceSelection := selector.AliveDecls() + origJS := renderPackage(t, origArchives[rootPath], false) + readJS := renderPackage(t, readArchives[rootPath], false) - sel := &selectionTester{ - DeclCode: &selectionCodeTester{ - t: t, - packages: packages, - dceSelection: dceSelection, - codeName: `DeclCode`, - getCode: func(d *Decl) []byte { return d.DeclCode }, - }, - InitCode: &selectionCodeTester{ - t: t, - packages: packages, - dceSelection: dceSelection, - codeName: `InitCode`, - getCode: func(d *Decl) []byte { return d.InitCode }, - }, + if diff := cmp.Diff(string(origJS), string(readJS)); diff != "" { + t.Errorf("the reloaded files produce different JS:\n%s", diff) } - - sel.DeclCode.IsDead(`^\s*Foo = \$newType`) - sel.DeclCode.IsDead(`^\s*Bar\[\d+ /\* int \*/\] = \$newType`) - sel.DeclCode.IsDead(`^\s*\$ptrType\(Bar\[\d+ /\* int \*/\]\)\.prototype\.Baz`) - sel.InitCode.IsDead(`ghost = new Bar\[\d+ /\* int \*/\]\.ptr\(7\)`) } func compareOrder(t *testing.T, sourceFiles []srctesting.Source, minify bool) { @@ -553,12 +509,68 @@ func compileProject(t *testing.T, root *packages.Package, minify bool) map[strin return archiveCache } +// newTime creates an arbitrary time.Time offset by the given number of seconds. +// This is useful for quickly creating times that are before or after another. +func newTime(seconds float64) time.Time { + return time.Date(1969, 7, 20, 20, 17, 0, 0, time.UTC). + Add(time.Duration(seconds * float64(time.Second))) +} + +// reloadCompiledProject persists the given archives into memory then reloads +// them from memory to simulate a cache reload of a precompiled project. +func reloadCompiledProject(t *testing.T, archives map[string]*Archive, rootPkgPath string) map[string]*Archive { + t.Helper() + + buildTime := newTime(5.0) + serialized := map[string][]byte{} + for path, a := range archives { + buf := &bytes.Buffer{} + if err := WriteArchive(a, buildTime, buf); err != nil { + t.Fatalf(`failed to write archive for %s: %v`, path, err) + } + serialized[path] = buf.Bytes() + } + + srcModTime := newTime(0.0) + reloadCache := map[string]*Archive{} + var importContext *ImportContext + importContext = &ImportContext{ + Packages: map[string]*types.Package{}, + Import: func(path string) (*Archive, error) { + // find in local cache + if a, ok := reloadCache[path]; ok { + return a, nil + } + + // deserialize archive + buf, ok := serialized[path] + if !ok { + t.Fatalf(`archive not found for %s`, path) + } + a, _, err := ReadArchive(path, bytes.NewReader(buf), srcModTime, importContext.Packages) + if err != nil { + t.Fatalf(`failed to read archive for %s: %v`, path, err) + } + reloadCache[path] = a + return a, nil + }, + } + + _, err := importContext.Import(rootPkgPath) + if err != nil { + t.Fatal(`failed to reload archives:`, err) + } + return reloadCache +} + func renderPackage(t *testing.T, archive *Archive, minify bool) []byte { t.Helper() - selection := make(map[*Decl]struct{}) + + sel := &dce.Selector[*Decl]{} for _, d := range archive.Declarations { - selection[d] = struct{}{} + sel.Include(d, false) } + selection := sel.AliveDecls() buf := &bytes.Buffer{} diff --git a/compiler/package.go b/compiler/package.go index 022d2df7b..2f6af9c6b 100644 --- a/compiler/package.go +++ b/compiler/package.go @@ -6,7 +6,6 @@ import ( "go/token" "go/types" "strings" - "time" "github.com/gopherjs/gopherjs/compiler/internal/analysis" "github.com/gopherjs/gopherjs/compiler/internal/dce" @@ -291,7 +290,6 @@ func Compile(importPath string, files []*ast.File, fileSet *token.FileSet, impor FileSet: srcs.FileSet, Minified: minify, GoLinknames: goLinknames, - BuildTime: time.Now(), }, nil }