integration pprof into go-control-plane for benchmarking and resource

profiling

Signed-off-by: Alec Holmes <[email protected]>

.gitignore

@@ -16,3 +16,9 @@ examples/dyplomat/dyplomat
 *_gen_test.go
 
 /envoy*.log
+
+# Benchmarking artifacts
+*.gz
+*.pprof
+/benchmarks/reports
+/benchmarks/pngs

Makefile

@@ -4,8 +4,8 @@
 # Variables
 #------------------------------------------------------------------------------
 
-SHELL 	:= /bin/bash
-BINDIR	:= bin
+SHELL 		:= /bin/bash
+BINDIR		:= bin
 PKG 		:= github.com/envoyproxy/go-control-plane
 
 .PHONY: build
@@ -19,6 +19,9 @@ clean:
 	@go mod tidy
 	@rm -rf $(BINDIR)
 	@rm -rf *.log
+	@rm -rf *.pprof
+	@rm -rf benchmarks/reports
+	@rm -rf benchmarks/pngs
 
 # TODO(mattklein123): See the note in TestLinearConcurrentSetWatch() for why we set -parallel here
 # This should be removed.
@@ -79,6 +82,18 @@ integration.xds.delta: $(BINDIR)/test $(BINDIR)/upstream
 integration.ads.delta: $(BINDIR)/test $(BINDIR)/upstream
 	env XDS=delta-ads build/integration.sh
 
+
+#------------------------------------------------------------------------------
+# Benchmarks utilizing intergration tests
+#------------------------------------------------------------------------------
+.PHONY: benchmark docker_benchmarks
+benchmark: $(BINDIR)/test $(BINDIR)/upstream
+	env XDS=xds build/integration.sh
+
+docker_benchmarks:
+	docker build --pull -f Dockerfile.ci . -t gcp_ci && \
+	docker run -v $$(pwd):/go-control-plane $$(tty -s && echo "-it" || echo) gcp_ci /bin/bash -c /go-control-plane/build/do_benchmarks.sh
+
 #--------------------------------------
 #-- example xDS control plane server
 #--------------------------------------

benchmarks/README.md

@@ -0,0 +1,65 @@
+# Benchmarks
+
+These benchmarks are designed to run off the integration test suite, and provide detailed profiling artifacts generated with pprof.
+
+## Prerequisites
+- [Go 1.16+](https://golang.org/dl/)
+- [Graphviz](https://graphviz.org/download/)
+- [pprof](https://github.com/google/pprof)
+- [Docker](https://www.docker.com/)
+
+## Running Locally
+
+To run the benchmarks locally, we take a similar apprach to the integration tests.
+
+### Requirements
+
+* Envoy binary `envoy` available: set `ENVOY` environment variable to the
+  location of the binary, or use the default value `/usr/local/bin/envoy`
+* `go-control-plane` builds successfully
+* Local installation of pprof and graphviz for profiler output
+
+### Steps
+
+To run the benchmark:
+```
+make benchmark MODE=0
+```
+
+There are 5 different modes all corresponding to various profiling outputs:
+```go
+const (
+	PPROF_CPU int = iota
+	PPROF_HEAP
+	PPROF_MUTEX
+	PPROF_BLOCK
+	PPROF_GOROUTINE
+)
+```
+To specifiy the mode, use `MODE` environment variable that corresponds to the output you wish to evaluate.
+
+The output file will be stored at the project root with .pprof extension (e.g. `cpu.pprof`).
+
+Run `pprof` tool like so which will open up a shell. From there, you can run command such as `web` to visualize the result:
+
+```bash
+go tool pprof cpu.pprof 
+(pprof) web
+```
+For more information, run `help`.
+
+## Running With Docker
+
+To run the benchmarks, we just require the prerequisite of docker and go.
+
+### Steps
+
+To run the benchmarks:
+
+```
+make docker_benchmarks
+```
+
+This will generate all profile artifacts in the `./benchmarks/reports`. Graphical profile anaylsis is located in `/.benchmarks/pngs`.
+
+For more information on how to interpret these reports/graphs, [click here](https://github.com/google/pprof/blob/master/doc/README.md#graphical-reports)

benchmarks/client/README.md

@@ -0,0 +1,3 @@
+# Benchmark Client
+
+This test client provides simulation of various workloads when communicating with the go-control-plane management server.

benchmarks/client/main.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"log"
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/envoyproxy/go-control-plane/benchmarks/client/xds"
+	"github.com/urfave/cli/v2"
+)
+
+func main() {
+	// Statically set the max procs
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	app := &cli.App{
+		Name:  "xds-benchmark",
+		Usage: "xds benchmarking tool that simulates client workload",
+		Commands: []*cli.Command{
+			{
+				Name:    "run",
+				Aliases: []string{"r"},
+				Usage:   "run the benchmark with the provided duration",
+				Action: func(c *cli.Context) error {
+					arg := c.Args().First()
+					dur, err := time.ParseDuration(arg)
+					if err != nil {
+						return err
+					}
+
+					sess, err := xds.NewSession("localhost:50000")
+					if err != nil {
+						return err
+					}
+
+					return sess.Simulate(dur)
+				},
+			},
+		},
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

benchmarks/client/xds/opts.go

@@ -0,0 +1,48 @@
+package xds
+
+// Options are configuration settings for the discovery object
+type Options struct {
+	NodeID        string
+	Zone          string
+	Cluster       string
+	ResourceNames []string // List of Envoy resource names to subscribe to
+	ResourceType  string   // ex: type.googleapis.com/envoy.api.v2.ClusterLoadAssignment
+}
+
+// Option follows the functional opts pattern
+type Option func(*Options)
+
+// WithNode will inject the node id into the configuration object
+func WithNode(id string) Option {
+	return func(o *Options) {
+		o.NodeID = id
+	}
+}
+
+// WithZone will specificy which zone to use in the xDS discovery request
+func WithZone(zone string) Option {
+	return func(o *Options) {
+		o.Zone = zone
+	}
+}
+
+// WithCluster will specificy which cluster the request is announcing as
+func WithCluster(cluster string) Option {
+	return func(o *Options) {
+		o.Cluster = cluster
+	}
+}
+
+// WithResourceNames will inject a list of resources the user wants to place watches on
+func WithResourceNames(names []string) Option {
+	return func(o *Options) {
+		o.ResourceNames = names
+	}
+}
+
+// WithResourceType will inject the specific resource type that a user wants to stream
+func WithResourceType(resource string) Option {
+	return func(o *Options) {
+		o.ResourceType = resource
+	}
+}

benchmarks/client/xds/xds.go

@@ -0,0 +1,45 @@
+package xds
+
+import (
+	"time"
+
+	"google.golang.org/grpc"
+)
+
+// Sess holds a grpc connection as well as config options to use during the simulation
+type Sess struct {
+	Session *grpc.ClientConn
+	Opts    Options
+}
+
+// NewSession will dial a new benchmarking session with the configured options
+func NewSession(url string, opts ...Option) (*Sess, error) {
+	var options Options
+	for _, o := range opts {
+		o(&options)
+	}
+
+	conn, err := grpc.Dial(url, grpc.WithBlock(), grpc.WithInsecure())
+	if err != nil {
+		return nil, err
+	}
+
+	return &Sess{
+		Session: conn,
+		Opts:    options,
+	}, nil
+}
+
+// Simulate will start an xDS stream which provides simulatest clients communicating with an xDS server
+func (s *Sess) Simulate(target time.Duration) error {
+	// Create a loop that will continually do work until the elapsed time as passed
+	for timeout := time.After(target); ; {
+		select {
+		case <-timeout:
+			return nil
+		default:
+			// Do some work
+
+		}
+	}
+}

benchmarks/client/xds/xds_test.go

@@ -0,0 +1 @@
+package xds

build/do_benchmarks.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+go install golang.org/x/tools/cmd/goimports@latest
+go get -u github.com/google/pprof
+apt update && apt install -y graphviz
+
+cd /go-control-plane
+
+NUM_DIFF_LINES=`/go/bin/goimports -local github.com/envoyproxy/go-control-plane -d pkg | wc -l`
+if [[ $NUM_DIFF_LINES > 0 ]]; then
+  echo "Failed format check. Run make format"
+  exit 1
+fi
+
+make benchmark MODE=0
+make benchmark MODE=1
+make benchmark MODE=2
+make benchmark MODE=3
+make benchmark MODE=4
+
+mkdir -p benchmarks/reports
+mkdir -p benchmarks/pngs
+
+# generate our consumable pprof profiles
+pprof -text bin/test cpu.pprof > benchmarks/reports/cpu_text.txt
+pprof -tree bin/test cpu.pprof > benchmarks/reports/cpu_tree.txt
+pprof -traces bin/test cpu.pprof > benchmarks/reports/cpu_trace.txt
+
+pprof -text bin/test goroutine.pprof > benchmarks/reports/goroutine_text.txt
+pprof -tree bin/test goroutine.pprof > benchmarks/reports/goroutine_tree.txt
+pprof -traces bin/test goroutine.pprof > benchmarks/reports/goroutine_trace.txt
+
+pprof -text bin/test block.pprof > benchmarks/reports/block_text.txt
+pprof -tree bin/test block.pprof > benchmarks/reports/block_tree.txt
+pprof -traces bin/test block.pprof > benchmarks/reports/block_trace.txt
+
+pprof -text bin/test mutex.pprof > benchmarks/reports/mutex_text.txt
+pprof -tree bin/test mutex.pprof > benchmarks/reports/mutex_tree.txt
+pprof -traces bin/test mutex.pprof > benchmarks/reports/mutex_trace.txt
+
+pprof -text bin/test mem.pprof > benchmarks/reports/mem_text.txt
+pprof -tree bin/test mem.pprof > benchmarks/reports/mem_tree.txt
+pprof -traces bin/test mem.pprof > benchmarks/reports/mem_trace.txt
+
+# generate some cool pprof graphs
+pprof -png bin/test cpu.pprof > benchmarks/pngs/cpu_graph.png
+pprof -png bin/test goroutine.pprof > benchmarks/pngs/goroutine_graph.png
+pprof -png bin/test block.pprof > benchmarks/pngs/block_graph.png
+pprof -png bin/test mutex.pprof > benchmarks/pngs/mutex_graph.png
+pprof -png bin/test mem.pprof > benchmarks/pngs/mem_graph.png

build/integration.sh

@@ -13,8 +13,8 @@ MESSAGE=$'Hi, there!\n'
 # Management server type. Valid values are "ads", "xds", "rest", "delta", or "delta-ads"
 XDS=${XDS:-ads}
 
-# pprof profiler. True means turn on profiling
-PPROF=${PPROF:-false}
+# pprof profiler mode
+MODE=${MODE:-0}
 
 # Number of RTDS layers.
 if [ "$XDS" = "ads" ]; then
@@ -43,4 +43,4 @@ function cleanup() {
 trap cleanup EXIT
 
 # run the test suite (which also contains the control plane)
-bin/test --xds=${XDS} --runtimes=${RUNTIMES} --pprof=${PPROF} -debug -message="$MESSAGE" "$@"
+bin/test --xds=${XDS} --runtimes=${RUNTIMES} --pprofMode=${MODE} -debug -message="$MESSAGE" "$@"

go.mod

@@ -8,17 +8,24 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.9.1
 	github.com/golang/protobuf v1.5.2
 	github.com/google/go-cmp v0.5.9
+	github.com/pkg/profile v1.7.0
 	github.com/prometheus/client_model v0.3.0
 	github.com/stretchr/testify v1.8.1
+	github.com/urfave/cli/v2 v2.24.1
 	go.opentelemetry.io/proto/otlp v0.19.0
 	google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6
 	google.golang.org/grpc v1.52.0
 	google.golang.org/protobuf v1.28.1
 )
 
 require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/felixge/fgprof v0.9.3 // indirect
+	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/net v0.4.0 // indirect
 	golang.org/x/sys v0.3.0 // indirect
 	golang.org/x/text v0.5.0 // indirect