Implement configs uploader GCB job (bazelbuild#937)

* Implement configs uploader GCB job

* Add comments

* Address review comments

cloudbuild.yaml

@@ -1,30 +1,47 @@
+# Generate Bazel C++ & Java toolchain configs for the latest available Bazel version & Ubuntu 16.04
+# based RBE toolchain container (l.gcr.io/google/rbe-ubuntu16-04:latest) and upload the generated
+# configs to GCS.
 steps:
-# Build the RBE Config generator using Go 1.15 for a few OS/arch combinations.
+# Build the RBE Config generator & uploader using Go 1.15.
 - name: golang:1.15
-  env:
-  - 'GOOS=linux'
-  - 'GOARCH=amd64'
   args:
   - go
   - build
   - -o
-  - rbe_configs_gen_linux_amd64
+  - rbe_configs_gen
   - github.com/bazelbuild/bazel-toolchains/cmd/rbe_configs_gen
 - name: golang:1.15
-  env:
-  - 'GOOS=windows'
-  - 'GOARCH=amd64'
   args:
   - go
   - build
   - -o
-  - rbe_configs_gen_windows_amd64.exe
-  - github.com/bazelbuild/bazel-toolchains/cmd/rbe_configs_gen
+  - rbe_configs_upload
+  - github.com/bazelbuild/bazel-toolchains/cmd/rbe_configs_upload
+
+# Run the configs generator in the GCB docker builder because the config generator will launch a
+# docker container to generate C++ configs.
+- name: gcr.io/cloud-builders/docker
+  entrypoint: ./rbe_configs_gen
+  # We don't specify a Bazel version which should make the tool generate configs for the latest
+  # available Bazel release.
+  args:
+    # Latest available RBE Toolchain docker image.
+  - --toolchain_container=l.gcr.io/google/rbe-ubuntu16-04:latest
+  - --exec_os=linux
+  - --target_os=linux
+  - --output_tarball=rbe_default.tar
+  - --output_manifest=manifest.json
+
+# Publish the generated configs to GCS. The configs are *always* published, regardless of whether
+# the contents of the configs tarball have changed. The manifest always includes an update because
+# it includes the upload timestamp. See the docs at the top of rbe_configs_upload.go for details on
+# where configs are uploaded on GCS.
+#
+# This step doesn't need docker but just picking the same container as above.
+- name: gcr.io/cloud-builders/docker
+  entrypoint: ./rbe_configs_upload
+  args:
+  - --configs_tarball=rbe_default.tar
+  - --configs_manifest=manifest.json
 
-# Upload all built Go binaries to our publicly readable GCS bucket. These binaries are uploaded
-# for convenience only without any guarantees on their availability or functionality.
-- name: 'gcr.io/cloud-builders/gsutil'
-  args: ['cp', 'rbe_configs_gen_linux_amd64', 'gs://rbe-bazel-toolchains/cli-tools/latest/rbe_configs_gen_linux_amd64']
-- name: 'gcr.io/cloud-builders/gsutil'
-  args: ['cp', 'rbe_configs_gen_windows_amd64.exe', 'gs://rbe-bazel-toolchains/cli-tools/latest/rbe_configs_gen_windows_amd64.exe']
 

cmd/rbe_configs_gen/rbe_configs_gen.go

@@ -22,7 +22,7 @@ var (
 	outputTarball    = flag.String("output_tarball", "", "(Optional) Path where a tarball with the generated configs will be created.")
 	outputSrcRoot    = flag.String("output_src_root", "", "(Optional) Path to root directory of Bazel repository where generated configs should be copied to. Configs aren't copied if this is blank. Use '.' to specify the current directory.")
 	outputConfigPath = flag.String("output_config_path", "", "(Optional) Path relative to what was specified to --output_src_root where configs will be extracted. Defaults to root if unspecified. --output_src_root is mandatory if this argument is specified.")
-	outputManifest   = flag.String("output_manifest", "", "(Optional) Generate a text file with details about the generated configs.")
+	outputManifest   = flag.String("output_manifest", "", "(Optional) Generate a JSON file with details about the generated configs.")
 
 	// Optional input arguments that affect config generation for either C++ or Java configs.
 	genCppConfigs  = flag.Bool("generate_cpp_configs", true, "(Optional) Generate C++ configs. Defaults to true.")

cmd/rbe_configs_upload/rbe_configs_upload.go

@@ -0,0 +1,152 @@
+// Binary rbe_configs_upload uploads the artifacts generated by rbe_configs_gen to GCS. This tool
+// is meant for internal use by the owners of this repository only.
+// This tool will upload the given configs tarball & manifest to the following paths on GCS:
+// - gs://rbe-bazel-toolchains/configs/latest
+// - - rbe_default.tar (The configs tarball)
+// - - manifest.json (The JSON manifest)
+// - gs://rbe-bazel-toolchains/configs/bazel_<version>/latest
+// - - rbe_default.tar (The configs tarball)
+// - - manifest.json (The JSON manifest)
+// This tool will upload the above files even if the config tarball hasn't changed. This can happen
+// if there's been no new Bazel release or toolchain container release since the last time this tool
+// was run. Thus, the above GCS artifacts are unstable in the sense that their contents can change
+// if either a new Bazel or toolchain container is released. This is to avoid users depending on
+// these GCS artifacts in production. Instead, users should copy the artifacts into a GCS bucket
+// or other remote location under their control.
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/bazelbuild/bazel-toolchains/pkg/rbeconfigsgen"
+)
+
+var (
+	configsTarball  = flag.String("configs_tarball", "", "Path to the configs tarball generated by rbe_configs_gen to be uploaded to GCS.")
+	configsManifest = flag.String("configs_manifest", "", "Path to the JSON manifest generated by rbe_configs_gen.")
+)
+
+// manifest is the metadata about the configs that'll be uploaded to GCS.
+type manifest struct {
+	// Wrap around the manifest produced by rbe_configs_gen.
+	rbeconfigsgen.Manifest
+	// UploadTime is the time this manifest was uploaded. For information only.
+	UploadTime time.Time `json:"upload_time"`
+}
+
+// manifestFromFile loads the JSON manifest (in the format produced by rbe_configs_gen) from the
+// given file and injects the current time in the returned manifest object.
+func manifestFromFile(filePath string) (*manifest, error) {
+	blob, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read manifest JSON file %q: %w", filePath, err)
+	}
+	m := &manifest{}
+	if err := json.Unmarshal(blob, m); err != nil {
+		return nil, fmt.Errorf("error parsing contents of manifest file %q as JSON: %w", filePath, err)
+	}
+	// Ensure the mandatory fields used by this binary were specified.
+	if len(m.BazelVersion) == 0 {
+		return nil, fmt.Errorf("manifest %q did not specify bazel version", filePath)
+	}
+	m.UploadTime = time.Now()
+	return m, nil
+}
+
+// storageClient represents the GCS client.
+type storageClient struct {
+	client *storage.Client
+	// bucketName is the GCS bucket all artifacts will be uploaded to.
+	bucketName string
+}
+
+func newStorage(ctx context.Context) (*storageClient, error) {
+	c, err := storage.NewClient(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &storageClient{
+		client:     c,
+		bucketName: "rbe-bazel-toolchains",
+	}, nil
+}
+
+// upload uploads the bytes represented by the given reader as the given GCS object name.
+func (s *storageClient) upload(ctx context.Context, r io.Reader, objectName string) error {
+	w := s.client.Bucket(s.bucketName).Object(objectName).NewWriter(ctx)
+	if _, err := io.Copy(w, r); err != nil {
+		return fmt.Errorf("error while uploading to GCS object %q: %w", objectName, err)
+	}
+	// The actual upload might happen after Close is called so we need to capture any errors.
+	if err := w.Close(); err != nil {
+		return fmt.Errorf("error finishing upload to GCS object %q: %w", objectName, err)
+	}
+	return nil
+}
+
+// uploadArtifacts uploads the given blob of bytes representing a JSON manifest and the configs
+// tarball at the given path to the given GCS directory.
+func (s *storageClient) uploadArtifacts(ctx context.Context, manifest []byte, tarballPath, remoteDir string) error {
+	f, err := os.Open(tarballPath)
+	if err != nil {
+		return fmt.Errorf("unable to open configs tarball file %q: %w", tarballPath, err)
+	}
+	defer f.Close()
+
+	if err := s.upload(ctx, bytes.NewBuffer(manifest), fmt.Sprintf("%s/manifest.json", remoteDir)); err != nil {
+		return fmt.Errorf("error uploading manifest to GCS: %w", err)
+	}
+
+	if err := s.upload(ctx, f, fmt.Sprintf("%s/rbe_default.tar", remoteDir)); err != nil {
+		return fmt.Errorf("error uploading configs tarball to GCS: %w", err)
+	}
+	return nil
+}
+
+func main() {
+	flag.Parse()
+
+	if len(*configsTarball) == 0 {
+		log.Fatalf("--configs_tarball was not specified.")
+	}
+	if len(*configsManifest) == 0 {
+		log.Fatalf("--configs_manifest was not specified.")
+	}
+
+	ctx := context.Background()
+	sc, err := newStorage(ctx)
+	if err != nil {
+		log.Fatalf("Failed to initialize the GCS client: %v", err)
+	}
+
+	m, err := manifestFromFile(*configsManifest)
+	if err != nil {
+		log.Fatalf("Error reading config manifest: %v", err)
+	}
+	manifestBlob, err := json.MarshalIndent(m, "", " ")
+	if err != nil {
+		log.Fatalf("Error converting manifest into JSON: %v", err)
+	}
+
+	uploadDirs := []string{
+		"configs/latest",
+		fmt.Sprintf("configs/bazel_%s/latest", m.BazelVersion),
+	}
+	for _, u := range uploadDirs {
+		if err := sc.uploadArtifacts(ctx, manifestBlob, *configsTarball, u); err != nil {
+			log.Fatalf("Error uploading configs to GCS bucket %s, directory %s: %v", sc.bucketName, u, err)
+		}
+		log.Printf("Configs published to GCS bucket %s, directory %s.", sc.bucketName, u)
+	}
+	log.Printf("Configs published successfully.")
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/bazelbuild/bazel-toolchains
 go 1.15
 
 require (
+	cloud.google.com/go/storage v1.6.0
 	github.com/bazelbuild/bazelisk v1.7.4
 	github.com/coreos/go-semver v0.3.0
 	github.com/google/go-containerregistry v0.4.0