From 05153c39baa6096c273b5a3a05fa8a5fdcd6d926 Mon Sep 17 00:00:00 2001 From: Misha Sakhnov Date: Wed, 16 Oct 2024 16:03:45 +0200 Subject: [PATCH] vm-builder: vm-builder builds arch images Add arm support to vm-builder without cross-compilation --- Makefile | 18 ++++++++++----- vm-builder/files/Dockerfile.img | 21 ++++++++++------- vm-builder/files/inittab | 2 +- vm-builder/main.go | 5 +++-- vm-deploy.yaml | 40 +++++++++++++++++++++++++++++++-- 5 files changed, 68 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 392baeaab..20593818c 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ GOOS ?= $(shell go env GOOS) # The target architecture for linux kernel. Possible values: amd64 or arm64. # Any other supported by linux kernel architecture could be added by introducing new build step into neonvm/hack/kernel/Dockerfile.kernel-builder KERNEL_TARGET_ARCH ?= amd64 - +TARGET_ARCH ?= amd64 # Get the currently used golang base path GOPATH=$(shell go env GOPATH) # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) @@ -184,6 +184,7 @@ docker-build-controller: docker-build-go-base ## Build docker image for NeonVM c docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runner docker build \ --tag $(IMG_RUNNER) \ + --build-arg TARGET_ARCH=$(TARGET_ARCH) \ --build-arg GO_BASE_IMG=$(GO_BASE_IMG) \ --file neonvm-runner/Dockerfile \ . @@ -192,6 +193,7 @@ docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runne docker-build-daemon: ## Build docker image for NeonVM daemon. docker build \ --tag $(IMG_DAEMON) \ + --build-arg TARGET_ARCH=$(TARGET_ARCH) \ --file neonvm-daemon/Dockerfile \ . @@ -223,7 +225,7 @@ docker-build-scheduler: docker-build-go-base ## Build docker image for (autoscal .PHONY: docker-build-examples docker-build-examples: bin/vm-builder ## Build docker images for testing VMs - ./bin/vm-builder -src postgres:15-bullseye -dst $(E2E_TESTS_VM_IMG) -spec tests/e2e/image-spec.yaml + ./bin/vm-builder -src postgres:15-bullseye -dst $(E2E_TESTS_VM_IMG) -spec tests/e2e/image-spec.yaml -target-arch linux/$(TARGET_ARCH) .PHONY: docker-build-pg16-disk-test docker-build-pg16-disk-test: bin/vm-builder ## Build a VM image for testing @@ -487,8 +489,14 @@ CODE_GENERATOR_VERSION ?= v0.28.12 KUTTL ?= $(LOCALBIN)/kuttl # k8s deps @ 1.28.3 -KUTTL_VERSION ?= v0.16.0 - +KUTTL_VERSION ?= v0.19.0 +ifeq ($(GOARCH), arm64) + KUTTL_ARCH = arm64 +else ifeq ($(GOARCH), amd64) + KUTTL_ARCH = x86_64 +else + $(error Unsupported architecture: $(GOARCH)) +endif KUBECTL ?= $(LOCALBIN)/kubectl KUBECTL_VERSION ?= v1.28.12 @@ -530,7 +538,7 @@ $(KUBECTL): $(LOCALBIN) .PHONY: kuttl kuttl: $(KUTTL) ## Download kuttl locally if necessary. $(KUTTL): $(LOCALBIN) - @test -s $(LOCALBIN)/kuttl || { curl -sfSLo $(KUTTL) https://github.com/kudobuilder/kuttl/releases/download/$(KUTTL_VERSION)/kubectl-kuttl_$(subst v,,$(KUTTL_VERSION))_$(GOOS)_$(shell uname -m) && chmod +x $(KUTTL); } + test -s $(LOCALBIN)/kuttl || { curl -sfSLo $(KUTTL) https://github.com/kudobuilder/kuttl/releases/download/$(KUTTL_VERSION)/kubectl-kuttl_$(subst v,,$(KUTTL_VERSION))_$(GOOS)_$(KUTTL_ARCH) && chmod +x $(KUTTL); } .PHONY: k3d k3d: $(K3D) ## Download k3d locally if necessary. diff --git a/vm-builder/files/Dockerfile.img b/vm-builder/files/Dockerfile.img index b72027975..990ae8617 100644 --- a/vm-builder/files/Dockerfile.img +++ b/vm-builder/files/Dockerfile.img @@ -9,16 +9,19 @@ USER root FROM {{.NeonvmDaemonImage}} AS neonvm-daemon-loader +FROM busybox:1.35.0-musl AS busybox-loader + FROM alpine:3.19 AS vm-runtime +ARG TARGET_ARCH +RUN set -e && mkdir -p /neonvm/bin /neonvm/runtime /neonvm/config # add busybox -ENV BUSYBOX_VERSION 1.35.0 +COPY --from=busybox-loader /bin/busybox /neonvm/bin/busybox + RUN set -e \ - && mkdir -p /neonvm/bin /neonvm/runtime /neonvm/config \ - && wget -q https://busybox.net/downloads/binaries/${BUSYBOX_VERSION}-x86_64-linux-musl/busybox -O /neonvm/bin/busybox \ - && chmod +x /neonvm/bin/busybox \ - && /neonvm/bin/busybox --install -s /neonvm/bin + chmod +x /neonvm/bin/busybox \ + && /neonvm/bin/busybox --install -s /neonvm/bin -COPY helper.move-bins.sh /helper.move-bins.sh + COPY helper.move-bins.sh /helper.move-bins.sh # add udevd and agetty (with shared libs) RUN set -e \ @@ -49,8 +52,10 @@ RUN set -e \ # Install vector.dev binary RUN set -e \ - && wget https://packages.timber.io/vector/0.26.0/vector-0.26.0-x86_64-unknown-linux-musl.tar.gz -O - \ - | tar xzvf - --strip-components 3 -C /neonvm/bin/ ./vector-x86_64-unknown-linux-musl/bin/vector + && ARCH=$( [ "$TARGET_ARCH" = "linux/arm64" ] && echo "aarch64" || echo "x86_64") \ + && wget https://packages.timber.io/vector/0.26.0/vector-${ARCH}-unknown-linux-musl.tar.gz -O - \ + | tar xzvf - --strip-components 3 -C /neonvm/bin/ ./vector-${ARCH}-unknown-linux-musl/bin/vector + # chrony RUN set -e \ diff --git a/vm-builder/files/inittab b/vm-builder/files/inittab index bc59e8404..9f7c284ec 100644 --- a/vm-builder/files/inittab +++ b/vm-builder/files/inittab @@ -12,5 +12,5 @@ {{ range .InittabCommands }} ::{{.SysvInitAction}}:su -p {{.CommandUser}} -c {{.ShellEscapedCommand}} {{ end }} -ttyS0::respawn:/neonvm/bin/agetty --8bits --local-line --noissue --noclear --noreset --host console --login-program /neonvm/bin/login --login-pause --autologin root 115200 ttyS0 linux +ttyAMA0::respawn:/neonvm/bin/agetty --8bits --local-line --noissue --noclear --noreset --host console --login-program /neonvm/bin/login --login-pause --autologin root 115200 ttyAMA0 linux ::shutdown:/neonvm/bin/vmshutdown diff --git a/vm-builder/main.go b/vm-builder/main.go index 6771e7dea..6fc354643 100644 --- a/vm-builder/main.go +++ b/vm-builder/main.go @@ -72,6 +72,7 @@ var ( version = flag.Bool("version", false, `Print vm-builder version`) daemonImageFlag = flag.String("daemon-image", "", `Specify the neonvm-daemon image: --daemon-image=neonvm-daemon:dev`) + targetArch = flag.String("target-arch", "linux/amd64", `Target architecture: --arch linux/amd64`) ) func AddTemplatedFileToTar(tw *tar.Writer, tmplArgs any, filename string, tmplString string) error { @@ -84,7 +85,6 @@ func AddTemplatedFileToTar(tw *tar.Writer, tmplArgs any, filename string, tmplSt if err = tmpl.Execute(&buf, tmplArgs); err != nil { return fmt.Errorf("failed to execute template for %q: %w", filename, err) } - return addFileToTar(tw, filename, buf.Bytes()) } @@ -129,7 +129,6 @@ type inittabCommand struct { func main() { flag.Parse() var dstIm string - if *version { fmt.Println(Version) os.Exit(0) @@ -366,6 +365,7 @@ func main() { buildArgs := make(map[string]*string) buildArgs["DISK_SIZE"] = size + buildArgs["TARGET_ARCH"] = targetArch opt := types.ImageBuildOptions{ AuthConfigs: authConfigs, Tags: []string{dstIm}, @@ -376,6 +376,7 @@ func main() { Dockerfile: "Dockerfile", Remove: true, ForceRemove: true, + Platform: *targetArch, } buildResp, err := cli.ImageBuild(ctx, tarBuffer, opt) if err != nil { diff --git a/vm-deploy.yaml b/vm-deploy.yaml index 9d787be9f..c6108d090 100644 --- a/vm-deploy.yaml +++ b/vm-deploy.yaml @@ -5,16 +5,34 @@ metadata: name: postgres16-disk-test annotations: # In this example, these bounds aren't necessary. So... here's what they look like :) - autoscaling.neon.tech/bounds: '{ "min": { "cpu": 0.25, "mem": "1Gi" }, "max": { "cpu": 1.25, "mem": "5Gi" } }' + autoscaling.neon.tech/bounds: '{ "min": { "cpu": 0.25, "mem": "1Gi" }, "max": { "cpu": 3, "mem": "5Gi" } }' labels: autoscaling.neon.tech/enabled: "true" # Set to "true" to continuously migrate the VM (TESTING ONLY) autoscaling.neon.tech/testing-only-always-migrate: "false" spec: + cpuScalingMode: sysfsScaling + terminationGracePeriodSeconds: 300 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + - ppc64le + - s390x + - key: kubernetes.io/os + operator: In + values: + - linux schedulerName: autoscale-scheduler enableSSH: true guest: - cpus: { min: 0.25, use: 0.25, max: 3 } + cpus: { min: 1, use: 1, max: 3} memorySlotSize: 1Gi memorySlots: { min: 1, use: 1, max: 5 } rootDisk: @@ -26,3 +44,21 @@ spec: - port: 10301 # monitor settings: swap: 1Gi + extraNetwork: + enable: true + disks: + - name: pgdata + mountPath: /var/lib/postgresql + emptyDisk: + size: 16Gi + - name: postgres-config + mountPath: /etc/postgresql + configMap: + name: example-config + items: + - key: postgresql.conf + path: postgresql.conf + - name: cache + mountPath: /neonvm/cache + tmpfs: + size: 1Gi