From af9c0a8a6c681340a78787831b25cbe8d4687ead Mon Sep 17 00:00:00 2001 From: Caleb Brown Date: Thu, 9 Dec 2021 12:56:08 +1100 Subject: [PATCH] Support tagged releases for images (#194) * Tag images built in Cloud Build so the images used are consistent. * Pass through the tag so that the commands can use it during analysis. --- build/build_docker.sh | 13 +++++++++++-- build/cloudbuild.yaml | 2 ++ cmd/analyze/Dockerfile | 3 +++ cmd/analyze/main.go | 11 ++++++++--- cmd/worker/main.go | 13 ++++++++----- infra/terraform/analysis.tf | 10 +++++++++- infra/terraform/build/main.tf | 16 ++++++++++++++++ infra/terraform/build/variables.tf | 3 +++ infra/terraform/terraform.tfvars | 6 ++++-- infra/terraform/variables.tf | 2 ++ internal/sandbox/sandbox.go | 18 ++++++++++++++++-- 11 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 infra/terraform/build/main.tf create mode 100644 infra/terraform/build/variables.tf diff --git a/build/build_docker.sh b/build/build_docker.sh index 9f43481f..60db8db9 100755 --- a/build/build_docker.sh +++ b/build/build_docker.sh @@ -1,6 +1,7 @@ #!/bin/bash -ex nopush=${NOPUSH:-"false"} +tag=${RELEASE_TAG} BASE_PATH="$(dirname $(dirname $(realpath $0)))" REGISTRY=gcr.io/ossf-malware-analysis @@ -10,7 +11,11 @@ declare -A ANALYSIS_IMAGES=( [node]=npm [python]=pypi [ruby]=rubygems ) pushd "$BASE_PATH/sandboxes" for image in "${!ANALYSIS_IMAGES[@]}"; do - docker build -t $REGISTRY/$image ${ANALYSIS_IMAGES[$image]} + extra_args="" + if [ "$tag" != "" ]; then + extra_args="-t $REGISTRY/$image:$tag" + fi + docker build $extra_args -t $REGISTRY/$image ${ANALYSIS_IMAGES[$image]} [[ "$nopush" == "false" ]] && docker push $REGISTRY/$image done popd @@ -20,7 +25,11 @@ declare -A CMD_IMAGES=( [analysis]=analyze [scheduler]=scheduler ) pushd "$BASE_PATH" for image in "${!CMD_IMAGES[@]}"; do - docker build -t $REGISTRY/$image -f cmd/${CMD_IMAGES[$image]}/Dockerfile . + extra_args="" + if [ "$tag" != "" ]; then + extra_args="-t $REGISTRY/$image:$tag --build-arg=SANDBOX_IMAGE_TAG=$tag" + fi + docker build $extra_args -t $REGISTRY/$image -f cmd/${CMD_IMAGES[$image]}/Dockerfile . [[ "$nopush" == "false" ]] && docker push $REGISTRY/$image done popd diff --git a/build/cloudbuild.yaml b/build/cloudbuild.yaml index 97b6a0b7..01c64243 100644 --- a/build/cloudbuild.yaml +++ b/build/cloudbuild.yaml @@ -1,5 +1,7 @@ steps: - name: 'gcr.io/cloud-builders/docker' + env: + - 'RELEASE_TAG=$TAG_NAME' entrypoint: bash dir: build args: ['-ex', 'build_docker.sh'] diff --git a/cmd/analyze/Dockerfile b/cmd/analyze/Dockerfile index 2c9fd151..af840f4a 100644 --- a/cmd/analyze/Dockerfile +++ b/cmd/analyze/Dockerfile @@ -26,3 +26,6 @@ RUN curl -fsSL https://gvisor.dev/archive.key | apt-key add - && \ COPY --from=build /src/analyze /usr/local/bin/analyze COPY --from=build /src/worker /usr/local/bin/worker + +ARG SANDBOX_IMAGE_TAG +ENV OSSF_SANDBOX_IMAGE_TAG=${SANDBOX_IMAGE_TAG} \ No newline at end of file diff --git a/cmd/analyze/main.go b/cmd/analyze/main.go index f587d69d..b8c141dd 100644 --- a/cmd/analyze/main.go +++ b/cmd/analyze/main.go @@ -19,6 +19,7 @@ var ( version = flag.String("version", "", "version") upload = flag.String("upload", "", "bucket path for uploading results") noPull = flag.Bool("nopull", false, "disables pulling down sandbox images") + imageTag = flag.String("image-tag", "", "set a image tag") ) func parseBucketPath(path string) (string, string) { @@ -70,9 +71,13 @@ func main() { args := manager.Args("all", *pkg, *version, *localPkg) - // Prepare the sandbox to use ensuring we respect the -"nopull" option and - // any local package is mapped through. - sbOpts := make([]sandbox.Option, 0) + // Prepare the sandbox: + // - Always pass through the tag. An empty tag is the same as "latest". + // - Respect the "-nopull" option. + // - Ensure any local package is mapped through. + sbOpts := []sandbox.Option{ + sandbox.Tag(*imageTag), + } if *noPull { sbOpts = append(sbOpts, sandbox.NoPull()) } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 2e88759c..3bca88d3 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -32,7 +32,7 @@ const ( localPkgPathFmt = "/local/%s" ) -func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blob.Bucket, resultsBucket string) error { +func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blob.Bucket, resultsBucket, imageTag string) error { name := msg.Metadata["name"] if name == "" { log.Warn("name is empty") @@ -78,7 +78,9 @@ func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blo ) localPkgPath := "" - sbOpts := make([]sandbox.Option, 0) + sbOpts := []sandbox.Option{ + sandbox.Tag(imageTag), + } if pkgPath != "" { // Copy remote package path to temporary file. @@ -120,7 +122,7 @@ func handleMessage(ctx context.Context, msg *pubsub.Message, packagesBucket *blo return nil } -func messageLoop(ctx context.Context, subURL, packagesBucket, resultsBucket string) error { +func messageLoop(ctx context.Context, subURL, packagesBucket, resultsBucket, imageTag string) error { sub, err := pubsub.OpenSubscription(ctx, subURL) if err != nil { return err @@ -140,7 +142,7 @@ func messageLoop(ctx context.Context, subURL, packagesBucket, resultsBucket stri return fmt.Errorf("error receiving message: %w", err) } - if err := handleMessage(ctx, msg, pkgsBkt, resultsBucket); err != nil { + if err := handleMessage(ctx, msg, pkgsBkt, resultsBucket, imageTag); err != nil { log.Error("Failed to process message", "error", err) } @@ -153,10 +155,11 @@ func main() { subURL := os.Getenv("OSSMALWARE_WORKER_SUBSCRIPTION") packagesBucket := os.Getenv("OSSF_MALWARE_ANALYSIS_PACKAGES") resultsBucket := os.Getenv("OSSF_MALWARE_ANALYSIS_RESULTS") + imageTag := os.Getenv("OSSF_SANDBOX_IMAGE_TAG") log.Initalize(os.Getenv("LOGGER_ENV")) for { - err := messageLoop(ctx, subURL, packagesBucket, resultsBucket) + err := messageLoop(ctx, subURL, packagesBucket, resultsBucket, imageTag) if err != nil { if retryCount++; retryCount >= maxRetries { log.Error("Retries exceeded", diff --git a/infra/terraform/analysis.tf b/infra/terraform/analysis.tf index 518cc0c6..5618ddf6 100644 --- a/infra/terraform/analysis.tf +++ b/infra/terraform/analysis.tf @@ -13,5 +13,13 @@ terraform { module "docker_registry" { source = "./docker_registry" - project = var.project + project = var.project } + +module "build" { + source = "./build" + + project = var.project + github_owner = var.github_owner + github_repo = var.github_repo +} \ No newline at end of file diff --git a/infra/terraform/build/main.tf b/infra/terraform/build/main.tf new file mode 100644 index 00000000..4f77bb0c --- /dev/null +++ b/infra/terraform/build/main.tf @@ -0,0 +1,16 @@ +# Google Cloud Build Triggers + +resource "google_cloudbuild_trigger" "image-build-trigger" { + name = "image-build-trigger" + project = var.project + + github { + owner = var.github_owner + name = var.github_repo + push { + tag = "^rel-[0-9]+$" + } + } + + filename = "build/cloudbuild.yaml" +} diff --git a/infra/terraform/build/variables.tf b/infra/terraform/build/variables.tf new file mode 100644 index 00000000..a1bb7557 --- /dev/null +++ b/infra/terraform/build/variables.tf @@ -0,0 +1,3 @@ +variable "project" {} +variable "github_owner" {} +variable "github_repo" {} diff --git a/infra/terraform/terraform.tfvars b/infra/terraform/terraform.tfvars index ffaae762..baa217d9 100644 --- a/infra/terraform/terraform.tfvars +++ b/infra/terraform/terraform.tfvars @@ -1,2 +1,4 @@ -project = "ossf-malware-analysis" -region = "us-central1" +project = "ossf-malware-analysis" +region = "us-central1" +github_owner = "ossf" +github_repo = "package-analysis" \ No newline at end of file diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf index 19f29aca..8c2455a4 100644 --- a/infra/terraform/variables.tf +++ b/infra/terraform/variables.tf @@ -1,2 +1,4 @@ variable "project" {} variable "region" {} +variable "github_owner" {} +variable "github_repo" {} diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 4b2726b1..c2d90197 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -81,6 +81,7 @@ func (v volume) args() []string { // Implements the Sandbox interface using "podman". type podmanSandbox struct { image string + tag string init bool noPull bool volumes []volume @@ -93,6 +94,7 @@ func (o option) set(sb *podmanSandbox) { o(sb) } func New(image string, options ...Option) Sandbox { sb := &podmanSandbox{ image: image, + tag: "", init: false, noPull: false, volumes: make([]volume, 0), @@ -120,6 +122,10 @@ func Volume(src, dest string) Option { }) } +func Tag(tag string) Option { + return option(func(sb *podmanSandbox) { sb.tag = tag }) +} + func podmanPull(image string) error { args := []string{"pull", image} cmd := exec.Command(podmanBin, args...) @@ -168,6 +174,14 @@ func (s *podmanSandbox) extraArgs() []string { return args } +func (s *podmanSandbox) imageWithTag() string { + tag := "latest" + if s.tag != "" { + tag = s.tag + } + return fmt.Sprintf("%s:%s", s.image, tag) +} + // Initializes the Sandbox ready for running commands. // // The image supplied will be pulled if it hasn't already been. @@ -176,7 +190,7 @@ func (s *podmanSandbox) Init() error { return nil } if !s.noPull { - if err := podmanPull(s.image); err != nil { + if err := podmanPull(s.imageWithTag()); err != nil { return err } } @@ -200,7 +214,7 @@ func (s *podmanSandbox) Run(args ...string) (*RunResult, error) { return &RunResult{}, err } - cmd := podmanRunCmd(s.image, args, s.extraArgs()) + cmd := podmanRunCmd(s.imageWithTag(), args, s.extraArgs()) var stdout bytes.Buffer var stderr bytes.Buffer