From e1836efe88fda4f8a03d06d3d7ccea277fd66ded Mon Sep 17 00:00:00 2001 From: subham sarkar Date: Sat, 28 Sep 2024 01:27:56 +0530 Subject: [PATCH] super fast crossbuilds --- dev-tools/mage/build.go | 8 +-- dev-tools/mage/check.go | 2 +- dev-tools/mage/clean.go | 3 +- dev-tools/mage/common.go | 69 ++++++++----------- dev-tools/mage/config.go | 13 ++-- dev-tools/mage/crossbuild.go | 118 +++++++++++++++++++++----------- dev-tools/mage/dockerbuilder.go | 71 +++++++++---------- dev-tools/mage/platforms.go | 24 ++++--- 8 files changed, 168 insertions(+), 140 deletions(-) diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 263299671fd..bcfddef7d37 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "go/build" - "log" "os" "path/filepath" "strings" @@ -124,8 +123,7 @@ func DefaultGolangCrossBuildArgs() BuildArgs { // environment. func GolangCrossBuild(params BuildArgs) error { if os.Getenv("GOLANG_CROSSBUILD") != "1" { - return errors.New("Use the crossBuild target. golangCrossBuild can " + - "only be executed within the golang-crossbuild docker environment.") + return errors.New("use the crossBuild target. golangCrossBuild can only be executed within the golang-crossbuild docker environment") } defer DockerChown(filepath.Join(params.OutputDir, params.Name+binaryExtension(GOOS))) @@ -206,7 +204,7 @@ func Build(params BuildArgs) error { } if GOOS == "windows" && params.WinMetadata { - log.Println("Generating a .syso containing Windows file metadata.") + fmt.Println("Generating a .syso containing Windows file metadata.") syso, err := MakeWindowsSysoFile() if err != nil { return fmt.Errorf("failed generating Windows .syso metadata file: %w", err) @@ -214,7 +212,7 @@ func Build(params BuildArgs) error { defer os.Remove(syso) } - log.Println("Adding build environment vars:", env) + fmt.Println("Adding build environment vars:", env) return sh.RunWith(env, "go", args...) } diff --git a/dev-tools/mage/check.go b/dev-tools/mage/check.go index a9547634eb5..e01e98c16e1 100644 --- a/dev-tools/mage/check.go +++ b/dev-tools/mage/check.go @@ -253,7 +253,7 @@ func checkDashboardForErrors(file string, d []byte) bool { var dashboard DashboardObject err := json.Unmarshal(d, &dashboard) if err != nil { - fmt.Println(fmt.Sprintf("failed to parse dashboard from %s: %s", file, err)) + fmt.Printf("failed to parse dashboard from %s: %s\n", file, err) return true } diff --git a/dev-tools/mage/clean.go b/dev-tools/mage/clean.go index c58a7b56ab0..c8794b72066 100644 --- a/dev-tools/mage/clean.go +++ b/dev-tools/mage/clean.go @@ -53,8 +53,7 @@ func Clean(pathLists ...[]string) error { if err := sh.Rm(f); err != nil { if errors.Is(err, os.ErrPermission) || strings.Contains(err.Error(), "permission denied") { - fmt.Printf("warn: cannot delete %q: %v, proceeding anyway\n", - f, err) + fmt.Printf("warn: cannot delete %q: %v, proceeding anyway\n", f, err) continue } return err diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 1c1ca25d95b..2fc2122547d 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -459,7 +459,7 @@ func Tar(src string, targetFile string) error { func untar(sourceFile, destinationDir string) error { file, err := os.Open(sourceFile) if err != nil { - return err + return fmt.Errorf("failed to open source file: %w", err) } defer file.Close() @@ -467,7 +467,7 @@ func untar(sourceFile, destinationDir string) error { if strings.HasSuffix(sourceFile, ".gz") { if fileReader, err = gzip.NewReader(file); err != nil { - return err + return fmt.Errorf("failed to create gzip reader: %w", err) } defer fileReader.Close() } @@ -480,38 +480,31 @@ func untar(sourceFile, destinationDir string) error { if err == io.EOF { break } - return err + return fmt.Errorf("error reading tar: %w", err) } path := filepath.Join(destinationDir, header.Name) - if !strings.HasPrefix(path, destinationDir) { + if !strings.HasPrefix(path, filepath.Clean(destinationDir)+string(os.PathSeparator)) { return fmt.Errorf("illegal file path in tar: %v", header.Name) } switch header.Typeflag { case tar.TypeDir: if err = os.MkdirAll(path, os.FileMode(header.Mode)); err != nil { - return err + return fmt.Errorf("failed to create directory: %w", err) } case tar.TypeReg: - writer, err := os.Create(path) + writer, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode)) if err != nil { - return err - } - - if _, err = io.Copy(writer, tarReader); err != nil { - return err - } - - if err = os.Chmod(path, os.FileMode(header.Mode)); err != nil { - return err + return fmt.Errorf("failed to create file: %w", err) } - - if err = writer.Close(); err != nil { - return err + _, err = io.Copy(writer, tarReader) + writer.Close() + if err != nil { + return fmt.Errorf("failed to write file contents: %w", err) } default: - return fmt.Errorf("unable to untar type=%c in file=%s", header.Typeflag, path) + return fmt.Errorf("unsupported tar entry type: %c for file: %s", header.Typeflag, path) } } @@ -585,7 +578,7 @@ func ParallelCtx(ctx context.Context, fns ...interface{}) { } var mu sync.Mutex - var errs []string + var errs []error var wg sync.WaitGroup for _, fw := range fnWrappers { @@ -594,7 +587,7 @@ func ParallelCtx(ctx context.Context, fns ...interface{}) { defer func() { if v := recover(); v != nil { mu.Lock() - errs = append(errs, fmt.Sprint(v)) + errs = append(errs, fmt.Errorf("%s", v)) mu.Unlock() } wg.Done() @@ -602,10 +595,10 @@ func ParallelCtx(ctx context.Context, fns ...interface{}) { }() waitStart := time.Now() parallelJobs() <- 1 - log.Println("Parallel job waited", time.Since(waitStart), "before starting.") + fmt.Printf("Parallel job waited %v before starting.\n", time.Since(waitStart)) if err := fw(ctx); err != nil { mu.Lock() - errs = append(errs, fmt.Sprint(err)) + errs = append(errs, err) mu.Unlock() } }(fw) @@ -613,7 +606,7 @@ func ParallelCtx(ctx context.Context, fns ...interface{}) { wg.Wait() if len(errs) > 0 { - panic(fmt.Errorf(strings.Join(errs, "\n"))) + panic(errors.Join(errs...)) } } @@ -675,7 +668,6 @@ func FindFilesRecursive(match func(path string, info os.FileInfo) bool) ([]strin } if !info.Mode().IsRegular() { - // continue return nil } @@ -696,31 +688,25 @@ func FileConcat(out string, perm os.FileMode, files ...string) error { defer f.Close() w := bufio.NewWriter(f) + defer w.Flush() - append := func(file string) error { + for _, file := range files { in, err := os.Open(file) if err != nil { - return err + return fmt.Errorf("failed to open input file %s: %w", file, err) } defer in.Close() if _, err := io.Copy(w, in); err != nil { - return err + return fmt.Errorf("failed to copy from %s: %w", file, err) } - - return nil } - for _, in := range files { - if err := append(in); err != nil { - return err - } + if err := w.Flush(); err != nil { + return fmt.Errorf("failed to flush writer: %w", err) } - if err = w.Flush(); err != nil { - return err - } - return f.Close() + return nil } // MustFileConcat invokes FileConcat and panics if an error occurs. @@ -748,11 +734,10 @@ func VerifySHA256(file string, hash string) error { expectedHash := strings.TrimSpace(hash) if computedHash != expectedHash { - return fmt.Errorf("SHA256 verification of %v failed. Expected=%v, "+ - "but computed=%v", f.Name(), expectedHash, computedHash) + return fmt.Errorf("SHA256 verification of %v failed. Expected=%v, computed=%v", f.Name(), expectedHash, computedHash) } - log.Println("SHA256 OK:", f.Name()) + fmt.Printf("SHA256 OK: %s\n", f.Name()) return nil } @@ -773,7 +758,7 @@ func CreateSHA512File(file string) error { computedHash := hex.EncodeToString(sum.Sum(nil)) out := fmt.Sprintf("%v %v", computedHash, filepath.Base(file)) - return ioutil.WriteFile(file+".sha512", []byte(out), 0644) + return os.WriteFile(file+".sha512", []byte(out), 0644) } // Mage executes mage targets in the specified directory. diff --git a/dev-tools/mage/config.go b/dev-tools/mage/config.go index 822e7f0f163..251db63eea8 100644 --- a/dev-tools/mage/config.go +++ b/dev-tools/mage/config.go @@ -21,7 +21,6 @@ import ( "bytes" "errors" "fmt" - "io/ioutil" "os" "path/filepath" "regexp" @@ -136,7 +135,7 @@ func makeConfigTemplate(destination string, mode os.FileMode, confParams ConfigF confFile = confParams.Docker tmplParams = map[string]interface{}{"Docker": true} default: - panic(fmt.Errorf("Invalid config file type: %v", typ)) + panic(fmt.Errorf("invalid config file type: %v", typ)) } // Build the dependencies. @@ -196,7 +195,7 @@ func makeConfigTemplate(destination string, mode os.FileMode, confParams ConfigF } } - data, err := ioutil.ReadFile(confFile.Template) + data, err := os.ReadFile(confFile.Template) if err != nil { return fmt.Errorf("failed to read config template %q: %w", confFile.Template, err) } @@ -265,7 +264,7 @@ type moduleFieldsYmlData []struct { } func readModuleFieldsYml(path string) (title string, useShort bool, err error) { - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { return "", false, err } @@ -302,7 +301,7 @@ func moduleDashes(name string) string { func GenerateModuleReferenceConfig(out string, moduleDirs ...string) error { var moduleConfigs []moduleConfigTemplateData for _, dir := range moduleDirs { - modules, err := ioutil.ReadDir(dir) + modules, err := os.ReadDir(dir) if err != nil { return err } @@ -327,7 +326,7 @@ func GenerateModuleReferenceConfig(out string, moduleDirs ...string) error { var data []byte for _, f := range files { - data, err = ioutil.ReadFile(f) + data, err = os.ReadFile(f) if err != nil { if os.IsNotExist(err) { continue @@ -365,5 +364,5 @@ func GenerateModuleReferenceConfig(out string, moduleDirs ...string) error { "Modules": moduleConfigs, }) - return ioutil.WriteFile(createDir(out), []byte(config), 0644) + return os.WriteFile(createDir(out), []byte(config), 0644) } diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 972531c25a8..d9de45cef09 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "go/build" - "log" "os" "path/filepath" "runtime" @@ -57,11 +56,9 @@ func init() { if packageTypes := os.Getenv("PACKAGES"); len(packageTypes) > 0 { for _, pkgtype := range strings.Split(packageTypes, ",") { var p PackageType - err := p.UnmarshalText([]byte(pkgtype)) - if err != nil { - continue + if err := p.UnmarshalText([]byte(pkgtype)); err == nil { + SelectedPackageTypes = append(SelectedPackageTypes, p) } - SelectedPackageTypes = append(SelectedPackageTypes, p) } } } @@ -112,9 +109,8 @@ func ImageSelector(f ImageSelectorFunc) func(params *crossBuildParams) { // AddPlatforms sets dependencies on others platforms. func AddPlatforms(expressions ...string) func(params *crossBuildParams) { return func(params *crossBuildParams) { - var list BuildPlatformList for _, expr := range expressions { - list = NewPlatformList(expr) + list := NewPlatformList(expr) params.Platforms = params.Platforms.Merge(list) } } @@ -136,7 +132,7 @@ func CrossBuild(options ...CrossBuildOption) error { } if len(params.Platforms) == 0 { - log.Printf("Skipping cross-build of target=%v because platforms list is empty.", params.Target) + fmt.Printf("Skipping cross-build of target=%v because platforms list is empty.\n", params.Target) return nil } @@ -147,14 +143,13 @@ func CrossBuild(options ...CrossBuildOption) error { if platform.GOOS() == "aix" { if len(params.Platforms) != 1 { return errors.New("AIX cannot be crossbuilt with other platforms. Set PLATFORMS='aix/ppc64'") - } else { - // This is basically a short-out so we can attempt to build on AIX in a relatively generic way - log.Printf("Target is building for AIX, skipping normal crossbuild process") - args := DefaultBuildArgs() - args.OutputDir = filepath.Join("build", "golang-crossbuild") - args.Name += "-" + Platform.GOOS + "-" + Platform.Arch - return Build(args) } + // This is basically a short-out so we can attempt to build on AIX in a relatively generic way + fmt.Printf("Target is building for AIX, skipping normal crossbuild process\n") + args := DefaultBuildArgs() + args.OutputDir = filepath.Join("build", "golang-crossbuild") + args.Name += "-" + Platform.GOOS + "-" + Platform.Arch + return Build(args) } } // If we're here, something isn't set. @@ -163,7 +158,7 @@ func CrossBuild(options ...CrossBuildOption) error { // Docker is required for this target. if err := HaveDocker(); err != nil { - return err + return fmt.Errorf("docker is required for crossbuild: %w", err) } if CrossBuildMountModcache { @@ -175,7 +170,7 @@ func CrossBuild(options ...CrossBuildOption) error { // Build the magefile for Linux, so we can run it inside the container. mg.Deps(buildMage) - log.Println("crossBuild: Platform list =", params.Platforms) + fmt.Printf("crossBuild: Platform list: %v\n", params.Platforms) var deps []interface{} for _, buildPlatform := range params.Platforms { if !buildPlatform.Flags.CanCrossBuild() { @@ -207,13 +202,26 @@ func CrossBuildXPack(options ...CrossBuildOption) error { return CrossBuild(o...) } -// buildMage pre-compiles the magefile to a binary using the GOARCH parameter. -// It has the benefit of speeding up the build because the -// mage -compile is done only once rather than in each Docker container. +// buildMage pre-compiles the magefile to a binary using the current GOARCH. +// It speeds up the build process by compiling mage only once for each architecture. func buildMage() error { - arch := runtime.GOARCH - return sh.RunWith(map[string]string{"CGO_ENABLED": "0"}, "mage", "-f", "-goos=linux", "-goarch="+arch, - "-compile", CreateDir(filepath.Join("build", "mage-linux-"+arch))) + const arch = runtime.GOARCH + + args := []string{ + "-f", + "-goos=linux", + "-goarch=" + arch, + "-compile", + filepath.Join("build", "mage-linux-"+arch), + } + + env := map[string]string{"CGO_ENABLED": "0"} + err := sh.RunWith(env, "mage", args...) + if err != nil { + return fmt.Errorf("failed to compile mage: %w", err) + } + + return nil } func CrossBuildImage(platform string) (string, error) { @@ -246,10 +254,10 @@ func CrossBuildImage(platform string) (string, error) { goVersion, err := GoVersion() if err != nil { - return "", err + return "", fmt.Errorf("failed to get Go version: %w", err) } - return BeatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil + return fmt.Sprintf("%s:%s-%s", BeatsCrossBuildImage, goVersion, tagSuffix), nil } // GolangCrossBuilder executes the specified mage target inside of the @@ -272,27 +280,29 @@ func (b GolangCrossBuilder) Build() error { mountPoint := filepath.ToSlash(filepath.Join("/go", "src", repoInfo.CanonicalRootImportPath)) // use custom dir for build if given, subdir if not: - cwd := repoInfo.SubDir - if b.InDir != "" { - cwd = b.InDir + cwd := b.InDir + if cwd == "" { + cwd = repoInfo.SubDir } workDir := filepath.ToSlash(filepath.Join(mountPoint, cwd)) builderArch := runtime.GOARCH - buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-"+builderArch)) + buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build", "mage-linux-"+builderArch)) if err != nil { - return fmt.Errorf("failed to determine mage-linux-"+builderArch+" relative path: %w", err) + return fmt.Errorf("failed to determine mage-linux-%s relative path: %w", builderArch, err) } dockerRun := sh.RunCmd("docker", "run") image, err := b.ImageSelector(b.Platform) if err != nil { - return fmt.Errorf("failed to determine golang-crossbuild image tag: %w", err) + return fmt.Errorf("failed to determine golang-crossbuild image tag for platform %s: %w", b.Platform, err) } + verbose := "" if mg.Verbose() { verbose = "true" } + var args []string // There's a bug on certain debian versions: // https://discuss.linuxcontainers.org/t/debian-jessie-containers-have-extremely-low-performance/1272 @@ -309,35 +319,65 @@ func (b GolangCrossBuilder) Build() error { "--env", "EXEC_GID="+strconv.Itoa(os.Getgid()), ) } + if versionQualified { args = append(args, "--env", "VERSION_QUALIFIER="+versionQualifier) } + if CrossBuildMountModcache { // Mount $GOPATH/pkg/mod into the container, read-only. hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod") args = append(args, "-v", hostDir+":/go/pkg/mod:ro") } - if b.Platform == "darwin/amd64" { + switch b.Platform { + case "darwin/amd64": fmt.Printf(">> %v: Forcing DEV=0 for %s: https://github.com/elastic/golang-crossbuild/issues/217\n", b.Target, b.Platform) args = append(args, "--env", "DEV=0") - } else { - args = append(args, "--env", fmt.Sprintf("DEV=%v", DevBuild)) + default: + args = append(args, "--env", "DEV="+strconv.FormatBool(DevBuild)) } + // To speed up cross-compilation, we need to persist the build cache so that subsequent builds + // for the same arch are faster (⚡). + // + // As we want to persist the build cache, we need to mount the cache directory to the Docker host. + // This is done by mounting the host directory to the container. + // + // Path of the cache directory on the host: + // /build/.go-build/ + // Example: /build/.go-build/linux/amd64 + // + // As per: https://docs.docker.com/engine/storage/bind-mounts/#differences-between--v-and---mount-behavior + // If the directory doesn't exist, Docker does not automatically create it for you, but generates an error. + // So, we need to create the directory before mounting it. + // + // Also, in the container, the cache directory is mounted to /root/.cache/go-build. + buildCacheHostDir := filepath.Join(repoInfo.RootDir, "build", ".go-build", b.Platform) + buildCacheContainerDir := "/root/.cache/go-build" + if err = os.MkdirAll(buildCacheHostDir, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", buildCacheHostDir, err) + } + + // Common arguments args = append(args, "--rm", "--env", "GOFLAGS=-mod=readonly -buildvcs=false", "--env", "MAGEFILE_VERBOSE="+verbose, "--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""), - "--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot), + "--env", "SNAPSHOT="+strconv.FormatBool(Snapshot), + + // To persist the build cache, we need to mount the cache directory to the Docker host. + // With docker run, mount types are: bind, volume and tmpfs. For our use case, we have + // decide to use the bind mount type. + "--mount", fmt.Sprintf("type=bind,source=%s,target=%s", buildCacheHostDir, buildCacheContainerDir), "-v", repoInfo.RootDir+":"+mountPoint, "-w", workDir, ) + // Image and build command arguments args = append(args, image, - // Arguments for docker crossbuild entrypoint. For details see // https://github.com/elastic/golang-crossbuild/blob/main/go1.17/base/rootfs/entrypoint.go. "--build-cmd", buildCmd+" "+b.Target, @@ -354,9 +394,9 @@ func DockerChown(path string) { uid, _ := strconv.Atoi(EnvOr("EXEC_UID", "-1")) gid, _ := strconv.Atoi(EnvOr("EXEC_GID", "-1")) if uid > 0 && gid > 0 { - log.Printf(">>> Fixing file ownership issues from Docker at path=%v", path) + fmt.Printf(">>> Fixing file ownership issues from Docker at path=%v\n", path) if err := chownPaths(uid, gid, path); err != nil { - log.Println(err) + fmt.Println(err) } } } @@ -366,7 +406,7 @@ func chownPaths(uid, gid int, path string) error { start := time.Now() numFixed := 0 defer func() { - log.Printf("chown took: %v, changed %d files", time.Since(start), numFixed) + fmt.Printf("chown took: %v, changed %d files\n", time.Since(start), numFixed) }() return filepath.Walk(path, func(name string, info os.FileInfo, err error) error { diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 2066670dc80..6f2c7ff20ed 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -43,7 +43,7 @@ type dockerBuilder struct { func newDockerBuilder(spec PackageSpec) (*dockerBuilder, error) { imageName, err := spec.ImageName() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get image name: %w", err) } buildDir := filepath.Join(spec.packageDir, "docker-build") @@ -63,7 +63,7 @@ func (b *dockerBuilder) Build() error { } if err := b.copyFiles(); err != nil { - return err + return fmt.Errorf("failed to copy files: %w", err) } if err := b.prepareBuild(); err != nil { @@ -71,13 +71,14 @@ func (b *dockerBuilder) Build() error { } tag, err := b.dockerBuild() - tries := 3 - for err != nil && tries != 0 { + + const maxRetries = 3 + const retryInterval = 10 * time.Second + + for retries := 0; err != nil && retries < maxRetries; retries++ { fmt.Println(">> Building docker images again (after 10 s)") - // This sleep is to avoid hitting the docker build issues when resources are not available. - time.Sleep(time.Second * 10) + time.Sleep(retryInterval) tag, err = b.dockerBuild() - tries -= 1 } if err != nil { return fmt.Errorf("failed to build docker: %w", err) @@ -123,7 +124,7 @@ func (b *dockerBuilder) copyFiles() error { func (b *dockerBuilder) prepareBuild() error { elasticBeatsDir, err := ElasticBeatsDir() if err != nil { - return err + return fmt.Errorf("failed to get ElasticBeatsDir: %w", err) } templatesDir := filepath.Join(elasticBeatsDir, "dev-tools/packaging/templates/docker") @@ -139,8 +140,7 @@ func (b *dockerBuilder) prepareBuild() error { ".tmpl", ) - err = b.ExpandFile(path, target, data) - if err != nil { + if err := b.ExpandFile(path, target, data); err != nil { return fmt.Errorf("expanding template '%s' to '%s': %w", path, target, err) } } @@ -148,15 +148,15 @@ func (b *dockerBuilder) prepareBuild() error { }) if err != nil { - return err + return fmt.Errorf("failed to walk templates directory: %w", err) } return b.expandDockerfile(templatesDir, data) } func isDockerFile(path string) bool { - path = filepath.Base(path) - return strings.HasPrefix(path, "Dockerfile") || strings.HasPrefix(path, "docker-entrypoint") + base := filepath.Base(path) + return strings.HasPrefix(base, "Dockerfile") || strings.HasPrefix(base, "docker-entrypoint") } func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]interface{}) error { @@ -170,18 +170,21 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in entrypoint = e } - type fileExpansion struct { + files := []struct { source string target string + }{ + {dockerfile, "Dockerfile.tmpl"}, + {entrypoint, "docker-entrypoint.tmpl"}, } - for _, file := range []fileExpansion{{dockerfile, "Dockerfile.tmpl"}, {entrypoint, "docker-entrypoint.tmpl"}} { + + for _, file := range files { target := strings.TrimSuffix( filepath.Join(b.buildDir, file.target), ".tmpl", ) path := filepath.Join(templatesDir, file.source) - err := b.ExpandFile(path, target, data) - if err != nil { + if err := b.ExpandFile(path, target, data); err != nil { return fmt.Errorf("expanding template '%s' to '%s': %w", path, target, err) } } @@ -192,7 +195,7 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in func (b *dockerBuilder) dockerBuild() (string, error) { tag := fmt.Sprintf("%s:%s", b.imageName, b.Version) if b.Snapshot { - tag = tag + "-SNAPSHOT" + tag += "-SNAPSHOT" } if repository, _ := b.ExtraVars["repository"]; repository != "" { tag = fmt.Sprintf("%s/%s", repository, tag) @@ -201,12 +204,10 @@ func (b *dockerBuilder) dockerBuild() (string, error) { } func (b *dockerBuilder) dockerSave(tag string) error { - if _, err := os.Stat(distributionsDir); os.IsNotExist(err) { - err := os.MkdirAll(distributionsDir, 0750) - if err != nil { - return fmt.Errorf("cannot create folder for docker artifacts: %+v", err) - } + if err := os.MkdirAll(distributionsDir, 0750); err != nil { + return fmt.Errorf("cannot create folder for docker artifacts: %w", err) } + // Save the container as artifact outputFile := b.OutputFile if outputFile == "" { @@ -214,46 +215,46 @@ func (b *dockerBuilder) dockerSave(tag string) error { "Name": b.imageName, }) if err != nil { - return err + return fmt.Errorf("failed to expand output file name: %w", err) } outputFile = filepath.Join(distributionsDir, outputTar) } + var stderr bytes.Buffer cmd := exec.Command("docker", "save", tag) cmd.Stderr = &stderr stdout, err := cmd.StdoutPipe() if err != nil { - return err + return fmt.Errorf("failed to get stdout pipe: %w", err) } + if err = cmd.Start(); err != nil { - return err + return fmt.Errorf("failed to start docker save command: %w", err) } - err = func() error { + if err := func() error { f, err := os.Create(outputFile) if err != nil { - return err + return fmt.Errorf("failed to create output file: %w", err) } defer f.Close() w := gzip.NewWriter(f) defer w.Close() - _, err = io.Copy(w, stdout) - if err != nil { - return err + if _, err = io.Copy(w, stdout); err != nil { + return fmt.Errorf("failed to copy docker save output: %w", err) } return nil - }() - if err != nil { + }(); err != nil { return err } if err = cmd.Wait(); err != nil { if errmsg := strings.TrimSpace(stderr.String()); errmsg != "" { - err = fmt.Errorf(err.Error()+": %w", errors.New(errmsg)) + err = fmt.Errorf("%w: %s", err, errmsg) } - return err + return fmt.Errorf("docker save command failed: %w", err) } if err = CreateSHA512File(outputFile); err != nil { diff --git a/dev-tools/mage/platforms.go b/dev-tools/mage/platforms.go index f2e835f687c..c15cdcffb0a 100644 --- a/dev-tools/mage/platforms.go +++ b/dev-tools/mage/platforms.go @@ -327,7 +327,7 @@ func newPlatformExpression(expr string) (*platformExpression, error) { // // By default, the initial set include only the platforms designated as defaults. // To add additional platforms to list use an addition term that is designated -// with a plug sign (e.g. "+netbsd" or "+linux/armv7"). Or you may use "+all" +// with a plus sign (e.g. "+netbsd" or "+linux/armv7"). Or you may use "+all" // to change the initial set to include all possible platforms then filter // from there (e.g. "+all linux windows"). // @@ -446,27 +446,33 @@ func (list BuildPlatformList) Filter(expr string) BuildPlatformList { return out.deduplicate() } -// Merge creates a new list with the two list merged. +// Merge creates a new list with the two lists merged. func (list BuildPlatformList) Merge(with BuildPlatformList) BuildPlatformList { - out := append(list, with...) + out := make(BuildPlatformList, 0, len(list)+len(with)) + out = append(out, list...) out = append(out, with...) return out.deduplicate() } // deduplicate removes duplicate platforms and sorts the list. func (list BuildPlatformList) deduplicate() BuildPlatformList { - set := map[string]BuildPlatform{} - for _, item := range list { - set[item.Name] = item + if len(list) <= 1 { + return list } - var out BuildPlatformList - for _, v := range set { - out = append(out, v) + seen := make(map[string]struct{}, len(list)) + out := make(BuildPlatformList, 0, len(list)) + + for _, item := range list { + if _, exists := seen[item.Name]; !exists { + seen[item.Name] = struct{}{} + out = append(out, item) + } } sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out }