From 53f854df9c5a7b9f0026d9f334999820a8f8b133 Mon Sep 17 00:00:00 2001 From: Henri Koski Date: Thu, 9 Nov 2023 12:52:43 +0200 Subject: [PATCH] WIP Add golang sub package --- go.go | 42 ++-------- golang/build.go | 199 ++++++++++++++++++++++++++++++++++++++++++++ golang/go.go | 37 ++++++++ golang/target/go.go | 73 ++++++++++++++++ golang/testing.go | 99 ++++++++++++++++++++++ 5 files changed, 417 insertions(+), 33 deletions(-) create mode 100644 golang/build.go create mode 100644 golang/go.go create mode 100644 golang/target/go.go create mode 100644 golang/testing.go diff --git a/go.go b/go.go index 6b70b8f..75fc37d 100644 --- a/go.go +++ b/go.go @@ -11,10 +11,9 @@ import ( "context" "fmt" "log" - "os" "path" - "strings" + "github.com/elisasre/mageutil/golang" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" ) @@ -34,29 +33,17 @@ type BuildInfo struct { // Go is shorthand for go executable provided by system. func Go(ctx context.Context, args ...string) error { - return GoWith(ctx, nil, args...) + return golang.Go(ctx, args...) } // GoWith is shorthand for go executable provided by system. func GoWith(ctx context.Context, env map[string]string, args ...string) error { - fmt.Printf("env: %s\n", env) - return sh.RunWithV(env, "go", args...) + return golang.GoWith(ctx, env, args...) } // Targets returns list of main pkgs under CmdDir. func Targets(ctx context.Context) ([]string, error) { - entries, err := os.ReadDir(CmdDir) - if err != nil { - return nil, err - } - - targets := make([]string, 0, len(entries)) - for _, e := range entries { - if e.IsDir() { - targets = append(targets, e.Name()) - } - } - return targets, nil + return golang.BuildTargets(ctx) } // BuildAll binaries for targets returned by utils.Targets using utils.Build. @@ -185,12 +172,7 @@ func Run(ctx context.Context, name string, args ...string) error { // GoList lists all packages in given target. func GoList(ctx context.Context, target string) ([]string, error) { - pkgsRaw, err := sh.Output("go", "list", target) - if err != nil { - return nil, err - } - pkgs := strings.Split(strings.ReplaceAll(pkgsRaw, "\r\n", ","), "\n") - return pkgs, nil + return golang.ListPackages(ctx, target) } // BinDir returns path in format of target/bin/{GOOS}/{GOARCH} @@ -212,7 +194,7 @@ func BinDir() (string, error) { // Deprecated: use Tidy instead func Ensure(ctx context.Context) error { log.Println("WARNING: Ensure is deprecated, use Tidy instead") - return Tidy(ctx) + return golang.Tidy(ctx) } // EnsureInSync checks that all dependencies are up to date @@ -220,22 +202,16 @@ func Ensure(ctx context.Context) error { // Deprecated: use TidyAndVerifyNoChanges instead func EnsureInSync(ctx context.Context) error { log.Println("WARNING: EnsureInSync is deprecated, use TidyAndVerifyNoChanges instead") - return TidyAndVerifyNoChanges(ctx) + return golang.TidyAndVerify(ctx) } // Tidy runs go mod tidy func Tidy(ctx context.Context) error { - return Go(ctx, "mod", "tidy") + return golang.Tidy(ctx) } // TidyAndVerifyNoChanges runs go mod tidy and verifies that there are no changes to go.mod or go.sum // useful in CI/CD pipelines to validate that dependencies match go.mod func TidyAndVerifyNoChanges(ctx context.Context) error { - if err := Tidy(ctx); err != nil { - return err - } - if err := Git(ctx, "diff", "--exit-code", "--", "go.mod", "go.sum"); err != nil { - return fmt.Errorf("go.mod and go.sum are not in sync. run `go mod tidy` and commit changes") - } - return nil + return golang.TidyAndVerify(ctx) } diff --git a/golang/build.go b/golang/build.go new file mode 100644 index 0000000..2042537 --- /dev/null +++ b/golang/build.go @@ -0,0 +1,199 @@ +// package golang provides util functions for [Magefile]. +// For usage please refer to [documentation] provided by Magefile. +// For autocompletions see [completions]. +// +// [Magefile]: https://magefile.org/ +// [documentation]: https://magefile.org/importing/ +// [completions]: https://github.com/elisasre/mageutil/tree/main/completions +package golang + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path" + "strings" + + "github.com/magefile/mage/sh" +) + +// BinDir is base directory for build outputs. +const BinDir = "./target/bin/" + +// BuildInfo contains relevant information about produced binary. +type BuildInfo struct { + BinPath string + GOOS string + GOARCH string +} + +type ( + BuildPlatform struct{ OS, Arch string } + BuildMatrix []BuildPlatform +) + +// DefaultBuildMatrix defines subset of cross-compile targets supported by Go. +var DefaultBuildMatrix = BuildMatrix{ + {OS: "linux", Arch: "amd64"}, + {OS: "linux", Arch: "arm64"}, + {OS: "darwin", Arch: "amd64"}, + {OS: "darwin", Arch: "arm64"}, + {OS: "windows", Arch: "amd64"}, +} + +// Build builds binary which is created under ./target/bin/{GOOS}/{GOARCH}/. +func Build(ctx context.Context, name string, buildArgs ...string) (BuildInfo, error) { + return BuildWith(ctx, nil, name, buildArgs...) +} + +// BuildWith injects given env and builds binary which is created under ./target/bin/{GOOS}/{GOARCH}/. +func BuildWith(ctx context.Context, env map[string]string, name string, buildArgs ...string) (BuildInfo, error) { + return build(ctx, env, BinDir, name, buildArgs...) +} + +// BuildForPlatform builds binary for wanted architecture and os. +func BuildForPlatform(ctx context.Context, goos, goarch, name string, buildArgs ...string) (BuildInfo, error) { + return BuildForPlatformWith(ctx, nil, goos, goarch, name, buildArgs...) +} + +// BuildForPlatform injects env, builds binary for wanted architecture and os. +func BuildForPlatformWith(ctx context.Context, env map[string]string, goos, goarch, name string, buildArgs ...string) (BuildInfo, error) { + if env == nil { + env = map[string]string{} + } + env["GOOS"] = goos + env["GOARCH"] = goarch + return BuildWith(ctx, env, name, buildArgs...) +} + +// WithSHA is a wrapper for build functions that adds SHA256Sum calculation. +func WithSHA(info BuildInfo, err error) (BuildInfo, error) { + if err != nil { + return BuildInfo{}, err + } + + return info, SHA256Sum(info.BinPath, info.BinPath+".sha256") +} + +func BuildForTesting(ctx context.Context, name string) (BuildInfo, error) { + env := map[string]string{ + "CGO_ENABLED": "1", + } + + pkgs, err := ListPackages(ctx, "./...") + if err != nil { + return BuildInfo{}, err + } + + args := []string{"-race", "-cover", "-covermode", "atomic", "-coverpkg=" + strings.Join(pkgs, ",")} + return build(ctx, env, TestBinDir, name, args...) +} + +// BuildFromMatrixWithSHA is a higher level build utility function doing cross compilation with sha calculation. +func BuildFromMatrixWithSHA(ctx context.Context, env map[string]string, matrix BuildMatrix, name string, buildArgs ...string) ([]BuildInfo, error) { + info := make([]BuildInfo, 0, len(matrix)) + for _, m := range matrix { + i, err := BuildForPlatformWith(ctx, env, m.OS, m.Arch, name, buildArgs...) + if err != nil { + return nil, err + } + info = append(info, i) + } + + return info, nil +} + +// BuildTargets returns list packages under ./cmd/. +func BuildTargets(ctx context.Context) ([]string, error) { + entries, err := os.ReadDir("./cmd/") + if err != nil { + return nil, err + } + + targets := make([]string, 0, len(entries)) + for _, e := range entries { + if e.IsDir() { + targets = append(targets, e.Name()) + } + } + return targets, nil +} + +// ListPackages lists all packages in given target. +func ListPackages(ctx context.Context, target string) ([]string, error) { + pkgsRaw, err := sh.Output("go", "list", target) + if err != nil { + return nil, err + } + pkgs := strings.Split(strings.ReplaceAll(pkgsRaw, "\r\n", ","), "\n") + return pkgs, nil +} + +// SHA256Sum calculates sum for input file and stores it in output file. +// Output should be compatible with sha256sum program. +func SHA256Sum(input, output string) error { + f, err := os.Open(input) + if err != nil { + return err + } + defer f.Close() + + data, err := io.ReadAll(f) + if err != nil { + return err + } + sum := sha256.Sum256(data) + hexSum := hex.EncodeToString(sum[:]) + + sumFile, err := os.Create(output) + if err != nil { + return err + } + defer f.Close() + + _, err = fmt.Fprintf(sumFile, "%s *%s\n", hexSum, input) + if err != nil { + return err + } + + return nil +} + +func build(ctx context.Context, env map[string]string, base, name string, buildArgs ...string) (BuildInfo, error) { + info, err := prepareBuildInfo(env, base, name) + if err != nil { + return BuildInfo{}, err + } + + args := append([]string{"build", "-o", info.BinPath}, append(buildArgs, "./cmd/"+name)...) + if err = GoWith(ctx, env, args...); err != nil { + return BuildInfo{}, err + } + + return info, nil +} +func prepareBuildInfo(env map[string]string, base, name string) (BuildInfo, error) { + goos, err := sh.Output("go", "env", "GOOS") + if err != nil { + return BuildInfo{}, err + } + + goarch, err := sh.Output("go", "env", "GOARCH") + if err != nil { + return BuildInfo{}, err + } + + if envOS, ok := env["GOOS"]; ok { + goos = envOS + } + + if envArch, ok := env["GOARCH"]; ok { + goarch = envArch + } + + binaryPath := path.Join(base, goos, goarch, name) + return BuildInfo{BinPath: binaryPath, GOOS: goos, GOARCH: goarch}, nil +} diff --git a/golang/go.go b/golang/go.go new file mode 100644 index 0000000..1bcee19 --- /dev/null +++ b/golang/go.go @@ -0,0 +1,37 @@ +package golang + +import ( + "context" + "fmt" + + "github.com/elisasre/mageutil/git" + "github.com/magefile/mage/sh" +) + +// Go is shorthand for go executable provided by system. +func Go(ctx context.Context, args ...string) error { + return GoWith(ctx, nil, args...) +} + +// GoWith is shorthand for go executable provided by system. +func GoWith(ctx context.Context, env map[string]string, args ...string) error { + fmt.Printf("env: %s\n", env) + return sh.RunWithV(env, "go", args...) +} + +// Tidy runs go mod tidy. +func Tidy(ctx context.Context) error { + return Go(ctx, "mod", "tidy") +} + +// TidyAndVerify runs go mod tidy and verifies that there are no changes to go.mod or go.sum. +// This is useful in CI/CD pipelines to validate that dependencies match go.mod. +func TidyAndVerify(ctx context.Context) error { + if err := Tidy(ctx); err != nil { + return err + } + if err := git.Git(ctx, "diff", "--exit-code", "--", "go.mod", "go.sum"); err != nil { + return fmt.Errorf("go.mod and go.sum are not in sync. run `go mod tidy` and commit changes") + } + return nil +} diff --git a/golang/target/go.go b/golang/target/go.go new file mode 100644 index 0000000..f4ca447 --- /dev/null +++ b/golang/target/go.go @@ -0,0 +1,73 @@ +package target + +import ( + "context" + + "github.com/elisasre/mageutil/golang" + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +var ( + BuildTarget = "" + RunArgs = []string{} + IntegrationTestArgs = []string{} +) + +type Go mg.Namespace + +// Build build binary and calculate sha sum for it +func (Go) Build(ctx context.Context) error { + return build(ctx, &golang.BuildInfo{}) +} + +// Run builds binary and executes it +func (Go) Run(ctx context.Context) { + var info golang.BuildInfo + mg.SerialCtxDeps(ctx, mg.F(build, &info), mg.F(run, info.BinPath)) +} + +// Test run unit and integration tests +func (Go) Test(ctx context.Context) { + mg.SerialCtxDeps(ctx, Go.UnitTest, Go.IntegrationTest) +} + +// UnitTest run all unit tests +func (Go) UnitTest(ctx context.Context) error { + return golang.UnitTest(ctx, golang.UnitTestCoverDir) +} + +// IntegrationTest run tests from ./integrationtests package +func (Go) IntegrationTest(ctx context.Context) error { + return golang.IntegrationTest(ctx, BuildTarget, golang.IntegrationTestPkg, golang.IntegrationTestCoverDir, IntegrationTestArgs...) +} + +// CoverProfile create coverage profile in text format +func (Go) CoverProfile(ctx context.Context) error { + return golang.CreateCoverProfile(ctx, golang.CoverProfileTxt, golang.IntegrationTestCoverDir, golang.UnitTestCoverDir) +} + +// ViewCoverage open test coverage in browser +func (Go) ViewCoverage(ctx context.Context) error { + return golang.Go(ctx, "tool", "cover", "-html", golang.CoverProfileTxt) +} + +// Tidy run go mod tidy +func (Go) Tidy(ctx context.Context) error { + return golang.Tidy(ctx) +} + +// TidyAndVerify verify that imports match go.mod +func (Go) TidyAndVerify(ctx context.Context) error { + return golang.TidyAndVerify(ctx) +} + +func build(ctx context.Context, info *golang.BuildInfo) error { + buildInfo, err := golang.WithSHA(golang.Build(ctx, BuildTarget)) + *info = buildInfo + return err +} + +func run(ctx context.Context, bin string) error { + return sh.RunV(bin, RunArgs...) +} diff --git a/golang/testing.go b/golang/testing.go new file mode 100644 index 0000000..9be5f55 --- /dev/null +++ b/golang/testing.go @@ -0,0 +1,99 @@ +package golang + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// Default values used by mageutil/golang/target.Go targets. +const ( + IntegrationTestPkg = "./integrationtests" + TestBinDir = "./target/tests/bin/" + UnitTestCoverDir = "./target/tests/cover/unit/" + IntegrationTestCoverDir = "./target/tests/cover/int/" + CoverProfileTxt = "./target/tests/cover/txt/cover.txt" +) + +// IntegrationTest executes integration tests in 4 phases: +// +// 1. Build application binary with coverage collection support. +// 2. Start application binary in background. +// 3. Execute integration tests. +// 4. Send SIGINT to application and wait for it to exit. +func IntegrationTest(ctx context.Context, name, testPkg, coverDir string, runArgs ...string) error { + buildInfo, err := BuildForTesting(ctx, name) + if err != nil { + return err + } + + stop, err := StartAppForIntegrationTests(ctx, buildInfo.BinPath, coverDir, runArgs...) + if err != nil { + return err + } + + err = RunIntegrationTests(ctx, testPkg) + if err != nil { + _ = stop() + return err + } + + return stop() +} + +// UnitTest runs all tests and collects coverage in coverDir. +func UnitTest(ctx context.Context, coverDir string) error { + err := os.MkdirAll(coverDir, 0755) + if err != nil { + return err + } + + dir, err := filepath.Abs(coverDir) + if err != nil { + return err + } + + env := map[string]string{"CGO_ENABLED": "1"} + return GoWith(ctx, env, "test", "-race", "-cover", "-covermode", "atomic", "./...", "-test.gocoverdir="+dir) +} + +// RunIntegrationTests runs tests inside given package with integration tag. +// To prevent caching -count=1 argument is also provided. +func RunIntegrationTests(ctx context.Context, integrationTestPkg string) error { + return Go(ctx, "test", "-tags=integration", "-count=1", integrationTestPkg) +} + +// StartAppForIntegrationTests starts application for integration testing in background. +func StartAppForIntegrationTests(ctx context.Context, bin, coverDir string, args ...string) (stop func() error, err error) { + err = os.MkdirAll(coverDir, 0755) + if err != nil { + return nil, err + } + + cmd := exec.Command(bin, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir) + + fmt.Printf("exec: %s %s\n", bin, args) + if err := cmd.Start(); err != nil { + return nil, err + } + + stop = func() error { + if err := cmd.Process.Signal(os.Interrupt); err != nil { + return err + } + return cmd.Wait() + } + + return stop, nil +} + +// CreateCoverProfile creates combined coverage profile in text format. +func CreateCoverProfile(ctx context.Context, output string, inputDirs ...string) error { + return Go(ctx, "tool", "covdata", "textfmt", "-i="+strings.Join(inputDirs, ","), "-o", CoverProfileTxt) +}