Merge pull request #176 from zeabur/buildkit

refactor(pkg): Use buildkit instead of docker

go.mod

@@ -26,7 +26,6 @@ require (
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
-	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
@@ -40,24 +39,25 @@ require (
 	golang.org/x/sys v0.12.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
-	github.com/Masterminds/semver v1.5.0
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/deckarep/golang-set v1.8.0
 	github.com/docker/distribution v2.8.2+incompatible
 	github.com/evanw/esbuild v0.19.4
 	github.com/gkampitakis/go-snaps v0.4.3
 	github.com/google/go-github/v53 v53.2.0
-	github.com/google/uuid v1.1.2
+	github.com/google/uuid v1.3.0
 	github.com/moznion/go-optional v0.10.0
 	github.com/otiai10/copy v1.12.0
 	github.com/pan93412/envexpander v1.1.0
 	github.com/samber/lo v1.38.1
 	github.com/samber/mo v1.8.0
+	github.com/spf13/cast v1.5.1
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/viper v1.16.0
 	golang.org/x/oauth2 v0.8.0

go.sum

@@ -38,8 +38,6 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
-github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
@@ -144,8 +142,9 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
@@ -541,8 +540,9 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

internal/nodejs/nextjs/main.go

@@ -21,7 +21,7 @@ import (
 
 // TransformServerless will transform build output of Next.js app to the serverless build output format of Zeabur
 // It is trying to implement the same logic as build function of https://github.com/vercel/vercel/tree/main/packages/next/src/index.ts
-func TransformServerless(image, workdir string) error {
+func TransformServerless(workdir string) error {
 
 	// create a tmpDir to store the build output of Next.js app
 	uuid := uuid2.New().String()
@@ -47,28 +47,11 @@ func TransformServerless(image, workdir string) error {
 
 	fmt.Println("=> Copying build output from image")
 
-	err := cp.Copy(workdir, tmpDir)
+	err := cp.Copy(path.Join(os.TempDir(), "/zbpack/buildkit"), path.Join(tmpDir))
 	if err != nil {
-		return err
+		return fmt.Errorf("copy buildkit output to tmp dir: %w", err)
 	}
 
-	err = utils.CopyFromImage(image, "/src/.next", tmpDir)
-	if err != nil {
-		return err
-	}
-
-	err = utils.CopyFromImage(image, "/src/node_modules", tmpDir)
-	if err != nil {
-		return err
-	}
-
-	err = utils.CopyFromImage(image, "/src/package.json", tmpDir)
-	if err != nil {
-		return err
-	}
-
-	_ = os.RemoveAll(path.Join(workdir, ".zeabur"))
-
 	serverlessFunctionPages := mapset.NewSet()
 	prerenderPaths := mapset.NewSet()
 	staticPages := mapset.NewSet()

internal/nodejs/node.go

@@ -20,6 +20,7 @@ type TemplateContext struct {
 
 	Framework  string
 	Serverless bool
+	OutputDir  string
 
 	Bun bool
 }
@@ -48,6 +49,7 @@ func getContextBasedOnMeta(meta types.PlanMeta) TemplateContext {
 		StartCmd:    meta["startCmd"],
 		Framework:   meta["framework"],
 		Serverless:  meta["serverless"] == "true",
+		OutputDir:   meta["outputDir"],
 
 		// The flag specific to planner/bun.
 		Bun: meta["bun"] == "true",

internal/nodejs/node_test.go

@@ -39,6 +39,8 @@ func TestGetContextBasedOnMeta_WithOutputdirAndSPAFramework(t *testing.T) {
 		InstallCmd:  "RUN npm install",
 		BuildCmd:    "npm run build",
 		StartCmd:    "npm run start",
+		Framework:   "wtfisthis",
+		OutputDir:   "dist",
 	})
 }
 
@@ -57,5 +59,7 @@ func TestGetContextBasedOnMeta_WithOutputdirAndMPAFramework(t *testing.T) {
 		InstallCmd:  "RUN npm install",
 		BuildCmd:    "npm run build",
 		StartCmd:    "npm run start",
+		Framework:   "hexo",
+		OutputDir:   "dist",
 	})
 }

internal/nodejs/nuxtjs/main.go

@@ -10,12 +10,11 @@ import (
 
 	uuid2 "github.com/google/uuid"
 	cp "github.com/otiai10/copy"
-	"github.com/zeabur/zbpack/internal/utils"
 	"github.com/zeabur/zbpack/pkg/types"
 )
 
 // TransformServerless will transform build output of Nuxt.js app to the serverless build output format of Zeabur
-func TransformServerless(image, workdir string) error {
+func TransformServerless(workdir string) error {
 
 	// create a tmpDir to store the build output of Next.js app
 	uuid := uuid2.New().String()
@@ -35,28 +34,11 @@ func TransformServerless(image, workdir string) error {
 
 	fmt.Println("=> Copying build output from image")
 
-	err := cp.Copy(workdir, tmpDir)
+	err := cp.Copy(path.Join(os.TempDir(), "zbpack/buildkit"), path.Join(tmpDir))
 	if err != nil {
 		return err
 	}
 
-	err = utils.CopyFromImage(image, "/src/.output", tmpDir)
-	if err != nil {
-		return err
-	}
-
-	err = utils.CopyFromImage(image, "/src/node_modules", tmpDir)
-	if err != nil {
-		return err
-	}
-
-	err = utils.CopyFromImage(image, "/src/package.json", tmpDir)
-	if err != nil {
-		return err
-	}
-
-	_ = os.RemoveAll(path.Join(workdir, ".zeabur"))
-
 	fmt.Println("=> Copying static asset files")
 
 	err = os.MkdirAll(path.Join(zeaburOutputDir, "static"), 0755)

internal/nodejs/templates/template.Dockerfile

@@ -19,13 +19,17 @@ RUN corepack enable && corepack prepare --all
 {{ .InstallCmd }}
 
 COPY . .
-
 {{ if and (eq .Framework "nuxt.js") .Serverless }}
 ENV NITRO_PRESET=node
 {{ end }}
-
 # Build if we can build it
 {{ if .BuildCmd }}RUN {{ .BuildCmd }}{{ end }}
-
+{{ if .Serverless }}
+FROM scratch as output
+COPY --from=build /src /
+{{ else if ne .OutputDir "" }}
+FROM scratch as output
+COPY --from=build /src/{{ .OutputDir }} /
+{{ else }}
 EXPOSE 8080
-CMD {{ .StartCmd }}
+CMD {{ .StartCmd }}{{ end }}

internal/static/TransformServerless.go

@@ -9,26 +9,19 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/zeabur/zbpack/internal/utils"
+	cp "github.com/otiai10/copy"
 	"github.com/zeabur/zbpack/pkg/types"
 )
 
 // TransformServerless copies the static files from output to .zeabur/output/static and creates a config.json file for SPA
-func TransformServerless(image, workdir string, meta types.PlanMeta, planType types.PlanType) error {
-	if planType == types.PlanTypeStatic {
-		err := utils.CopyFromImage(image, "/usr/share/nginx/html/static"+"/.", path.Join(workdir, ".zeabur/output/static"))
-		if err != nil {
-			return err
-		}
-	} else {
-		err := utils.CopyFromImage(image, path.Join("/src", meta["outputDir"])+"/.", path.Join(workdir, ".zeabur/output/static"))
-		if err != nil {
-			return err
-		}
+func TransformServerless(workdir string, meta types.PlanMeta) error {
+	err := cp.Copy(path.Join(os.TempDir(), "zbpack/buildkit", "/."), path.Join(workdir, ".zeabur/output/static"))
+	if err != nil {
+		return fmt.Errorf("copy static files from buildkit output to .zeabur/output/static: %w", err)
 	}
 
 	// delete hidden files and directories in output directory
-	err := deleteHiddenFilesAndDirs(path.Join(workdir, ".zeabur/output/static"))
+	err = deleteHiddenFilesAndDirs(path.Join(workdir, ".zeabur/output/static"))
 	if err != nil {
 		return fmt.Errorf("delete hidden files and directories in directory: %w", err)
 	}

internal/static/static.go

@@ -16,18 +16,14 @@ RUN apt-get update && apt-get install -y git
 COPY . .
 RUN hugo --minify
 
-FROM docker.io/library/nginx:alpine as runtime
-WORKDIR /usr/share/nginx/html/static
-COPY --from=builder /src/public .
-RUN echo "server { listen 8080; root /usr/share/nginx/html/static; absolute_redirect off; }"> /etc/nginx/conf.d/default.conf
-EXPOSE 8080`, nil
+FROM scratch as output
+COPY --from=builder /src/public /
+`, nil
 	}
 
-	dockerfile := `FROM docker.io/library/nginx:alpine as runtime
-WORKDIR /usr/share/nginx/html/static
-COPY . .
-RUN echo "server { listen 8080; root /usr/share/nginx/html/static; absolute_redirect off; location / { add_header 'Access-Control-Allow-Origin' '*'; if (\$request_method = 'OPTIONS') { return 204; } } }"> /etc/nginx/conf.d/default.conf
-EXPOSE 8080`
+	dockerfile := `FROM scratch as output
+COPY . /
+`
 
 	return dockerfile, nil
 }

internal/zbpack/zbpack.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"os/exec"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -56,6 +57,27 @@ func run(args []string) error {
 
 // build is used to build Docker image and show build plan.
 func build(path string) error {
+
+	// before start, check if buildctl is installed and buildkitd is running
+	err := exec.Command("buildctl", "debug", "workers").Run()
+	if err != nil {
+		red := "\033[31m"
+		blue := "\033[34m"
+		reset := "\033[0m"
+		gray := "\033[90m"
+
+		print(red, "buildctl is not installed or buildkitd is not running.\n", reset)
+		print("Learn more: https://github.com/moby/buildkit#quick-start\n\n", reset)
+		print(gray, "Or you can simply run the following command to run buildkitd in a container:\n", reset)
+		print(blue, "docker run -d --name buildkitd --privileged moby/buildkit:latest\n\n", reset)
+		print(gray, "And then install buildctl if you haven't:\n", reset)
+		print(blue, "docker cp buildkitd:/usr/bin/buildctl /usr/local/bin\n\n", reset)
+		print(gray, "After that, you can run zbpack again with the following command:\n", reset)
+		print(blue, "BUILDKIT_HOST=docker-container://buildkitd zbpack <...>\n", reset)
+
+		return nil
+	}
+
 	// TODO support online repositories
 	if strings.HasPrefix(path, "https://") || strings.HasPrefix(path, "http://") {
 		return fmt.Errorf("zbpack does not support building from online repositories yet")