Merge pull request gopherjs#1354 from Workiva/archiveSerial

Archive serialization updates and DCE fix

build/build.go

@@ -21,6 +21,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
@@ -938,23 +939,40 @@ func (s *Session) buildImportPathWithSrcDir(path string, srcDir string) (*Packag
 	return pkg, archive, nil
 }
 
+// getExeModTime will determine the mod time of the GopherJS binary
+// the first time this is called and cache the result for subsequent calls.
+var getExeModTime = func() func() time.Time {
+	var (
+		once   sync.Once
+		result time.Time
+	)
+	getTime := func() {
+		gopherjsBinary, err := os.Executable()
+		if err == nil {
+			var fileInfo os.FileInfo
+			fileInfo, err = os.Stat(gopherjsBinary)
+			if err == nil {
+				result = fileInfo.ModTime()
+				return
+			}
+		}
+		os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
+		result = time.Now()
+	}
+	return func() time.Time {
+		once.Do(getTime)
+		return result
+	}
+}()
+
 // BuildPackage compiles an already loaded package.
 func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
 	if archive, ok := s.UpToDateArchives[pkg.ImportPath]; ok {
 		return archive, nil
 	}
 
-	var fileInfo os.FileInfo
-	gopherjsBinary, err := os.Executable()
-	if err == nil {
-		fileInfo, err = os.Stat(gopherjsBinary)
-		if err == nil && fileInfo.ModTime().After(pkg.SrcModTime) {
-			pkg.SrcModTime = fileInfo.ModTime()
-		}
-	}
-	if err != nil {
-		os.Stderr.WriteString("Could not get GopherJS binary's modification timestamp. Please report issue.\n")
-		pkg.SrcModTime = time.Now()
+	if exeModTime := getExeModTime(); exeModTime.After(pkg.SrcModTime) {
+		pkg.SrcModTime = exeModTime
 	}
 
 	for _, importedPkgPath := range pkg.Imports {
@@ -966,22 +984,18 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
 			return nil, err
 		}
 
-		impModTime := importedPkg.SrcModTime
-		if impModTime.After(pkg.SrcModTime) {
+		if impModTime := importedPkg.SrcModTime; impModTime.After(pkg.SrcModTime) {
 			pkg.SrcModTime = impModTime
 		}
 	}
 
-	if pkg.FileModTime().After(pkg.SrcModTime) {
-		pkg.SrcModTime = pkg.FileModTime()
+	if fileModTime := pkg.FileModTime(); fileModTime.After(pkg.SrcModTime) {
+		pkg.SrcModTime = fileModTime
 	}
 
 	if !s.options.NoCache {
-		archive := s.buildCache.LoadArchive(pkg.ImportPath)
-		if archive != nil && !pkg.SrcModTime.After(archive.BuildTime) {
-			if err := archive.RegisterTypes(s.Types); err != nil {
-				panic(fmt.Errorf("failed to load type information from %v: %w", archive, err))
-			}
+		archive := s.buildCache.LoadArchive(pkg.ImportPath, pkg.SrcModTime, s.Types)
+		if archive != nil {
 			s.UpToDateArchives[pkg.ImportPath] = archive
 			// Existing archive is up to date, no need to build it from scratch.
 			return archive, nil
@@ -1021,7 +1035,7 @@ func (s *Session) BuildPackage(pkg *PackageData) (*compiler.Archive, error) {
 		fmt.Println(pkg.ImportPath)
 	}
 
-	s.buildCache.StoreArchive(archive)
+	s.buildCache.StoreArchive(archive, time.Now())
 	s.UpToDateArchives[pkg.ImportPath] = archive
 
 	return archive, nil

build/cache/cache.go

@@ -6,9 +6,11 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"go/build"
+	"go/types"
 	"os"
 	"path"
 	"path/filepath"
+	"time"
 
 	"github.com/gopherjs/gopherjs/compiler"
 	log "github.com/sirupsen/logrus"
@@ -90,7 +92,10 @@ func (bc BuildCache) String() string {
 
 // StoreArchive compiled archive in the cache. Any error inside this method
 // will cause the cache not to be persisted.
-func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
+//
+// The passed in buildTime is used to determine if the archive is out-of-date when reloaded.
+// Typically it should be set to the srcModTime or time.Now().
+func (bc *BuildCache) StoreArchive(a *compiler.Archive, buildTime time.Time) {
 	if bc == nil {
 		return // Caching is disabled.
 	}
@@ -106,7 +111,7 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
 		return
 	}
 	defer f.Close()
-	if err := compiler.WriteArchive(a, f); err != nil {
+	if err := compiler.WriteArchive(a, buildTime, f); err != nil {
 		log.Warningf("Failed to write build cache archive %q: %v", a, err)
 		// Make sure we don't leave a half-written archive behind.
 		os.Remove(f.Name())
@@ -125,7 +130,10 @@ func (bc *BuildCache) StoreArchive(a *compiler.Archive) {
 //
 // The returned archive would have been built with the same configuration as
 // the build cache was.
-func (bc *BuildCache) LoadArchive(importPath string) *compiler.Archive {
+//
+// The imports map is used to resolve package dependencies and may modify the
+// map to include the package from the read archive. See [gcexportdata.Read].
+func (bc *BuildCache) LoadArchive(importPath string, srcModTime time.Time, imports map[string]*types.Package) *compiler.Archive {
 	if bc == nil {
 		return nil // Caching is disabled.
 	}
@@ -140,12 +148,16 @@ func (bc *BuildCache) LoadArchive(importPath string) *compiler.Archive {
 		return nil // Cache miss.
 	}
 	defer f.Close()
-	a, err := compiler.ReadArchive(importPath, f)
+	a, buildTime, err := compiler.ReadArchive(importPath, f, srcModTime, imports)
 	if err != nil {
 		log.Warningf("Failed to read cached package archive for %q: %v", importPath, err)
 		return nil // Invalid/corrupted archive, cache miss.
 	}
-	log.Infof("Found cached package archive for %q, built at %v.", importPath, a.BuildTime)
+	if a == nil {
+		log.Infof("Found out-of-date package archive for %q, built at %v.", importPath, buildTime)
+		return nil // Archive is out-of-date, cache miss.
+	}
+	log.Infof("Found cached package archive for %q, built at %v.", importPath, buildTime)
 	return a
 }
 

build/cache/cache_test.go

@@ -1,7 +1,9 @@
 package cache
 
 import (
+	"go/types"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/gopherjs/gopherjs/compiler"
@@ -15,21 +17,24 @@ func TestStore(t *testing.T) {
 		Imports:    []string{"fake/dep"},
 	}
 
+	srcModTime := newTime(0.0)
+	buildTime := newTime(5.0)
+	imports := map[string]*types.Package{}
 	bc := BuildCache{}
-	if got := bc.LoadArchive(want.ImportPath); got != nil {
+	if got := bc.LoadArchive(want.ImportPath, srcModTime, imports); got != nil {
 		t.Errorf("Got: %s was found in the cache. Want: empty cache.", got.ImportPath)
 	}
-	bc.StoreArchive(want)
-	got := bc.LoadArchive(want.ImportPath)
+	bc.StoreArchive(want, buildTime)
+	got := bc.LoadArchive(want.ImportPath, srcModTime, imports)
 	if got == nil {
-		t.Errorf("Got: %s wan not found in the cache. Want: archive is can be loaded after store.", want.ImportPath)
+		t.Errorf("Got: %s was not found in the cache. Want: archive is can be loaded after store.", want.ImportPath)
 	}
 	if diff := cmp.Diff(want, got); diff != "" {
 		t.Errorf("Loaded archive is different from stored (-want,+got):\n%s", diff)
 	}
 
 	// Make sure the package names are a part of the cache key.
-	if got := bc.LoadArchive("fake/other"); got != nil {
+	if got := bc.LoadArchive("fake/other", srcModTime, imports); got != nil {
 		t.Errorf("Got: fake/other was found in cache: %#v. Want: nil for packages that weren't cached.", got)
 	}
 }
@@ -59,20 +64,54 @@ func TestInvalidation(t *testing.T) {
 		},
 	}
 
+	srcModTime := newTime(0.0)
+	buildTime := newTime(5.0)
+	imports := map[string]*types.Package{}
 	for _, test := range tests {
 		a := &compiler.Archive{ImportPath: "package/fake"}
-		test.cache1.StoreArchive(a)
+		test.cache1.StoreArchive(a, buildTime)
 
-		if got := test.cache2.LoadArchive(a.ImportPath); got != nil {
+		if got := test.cache2.LoadArchive(a.ImportPath, srcModTime, imports); got != nil {
 			t.Logf("-cache1,+cache2:\n%s", cmp.Diff(test.cache1, test.cache2))
 			t.Errorf("Got: %v loaded from cache. Want: build parameter change invalidates cache.", got)
 		}
 	}
 }
 
+func TestOldArchive(t *testing.T) {
+	cacheForTest(t)
+
+	want := &compiler.Archive{
+		ImportPath: "fake/package",
+		Imports:    []string{"fake/dep"},
+	}
+
+	buildTime := newTime(5.0)
+	imports := map[string]*types.Package{}
+	bc := BuildCache{}
+	bc.StoreArchive(want, buildTime)
+
+	oldSrcModTime := newTime(2.0) // older than archive build time, so archive is up-to-date
+	got := bc.LoadArchive(want.ImportPath, oldSrcModTime, imports)
+	if got == nil {
+		t.Errorf("Got: %s was nil. Want: up-to-date archive to be loaded.", want.ImportPath)
+	}
+
+	newerSrcModTime := newTime(7.0) // newer than archive build time, so archive is stale
+	got = bc.LoadArchive(want.ImportPath, newerSrcModTime, imports)
+	if got != nil {
+		t.Errorf("Got: %s was not nil. Want: stale archive to not be loaded.", want.ImportPath)
+	}
+}
+
 func cacheForTest(t *testing.T) {
 	t.Helper()
 	originalRoot := cacheRoot
 	t.Cleanup(func() { cacheRoot = originalRoot })
 	cacheRoot = t.TempDir()
 }
+
+func newTime(seconds float64) time.Time {
+	return time.Date(1969, 7, 20, 20, 17, 0, 0, time.UTC).
+		Add(time.Duration(seconds * float64(time.Second)))
+}