diff --git a/.gitignore b/.gitignore index 947b51001e..7996a18f43 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,9 @@ examples/dyplomat/dyplomat *_gen_test.go /envoy*.log + +# Benchmarking artifacts +*.gz +*.pprof +/benchmarks/reports +/benchmarks/pngs diff --git a/Makefile b/Makefile index b7d7db39d1..c4a4776e44 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,8 @@ # Variables #------------------------------------------------------------------------------ -SHELL := /bin/bash -BINDIR := bin +SHELL := /bin/bash +BINDIR := bin PKG := github.com/envoyproxy/go-control-plane .PHONY: build @@ -19,6 +19,9 @@ clean: @go mod tidy @rm -rf $(BINDIR) @rm -rf *.log + @rm -rf *.pprof + @rm -rf benchmarks/reports + @rm -rf benchmarks/pngs # TODO(mattklein123): See the note in TestLinearConcurrentSetWatch() for why we set -parallel here # This should be removed. @@ -79,6 +82,18 @@ integration.xds.delta: $(BINDIR)/test $(BINDIR)/upstream integration.ads.delta: $(BINDIR)/test $(BINDIR)/upstream env XDS=delta-ads build/integration.sh + +#------------------------------------------------------------------------------ +# Benchmarks utilizing intergration tests +#------------------------------------------------------------------------------ +.PHONY: benchmark docker_benchmarks +benchmark: $(BINDIR)/test $(BINDIR)/upstream + env XDS=xds build/integration.sh + +docker_benchmarks: + docker build --pull -f Dockerfile.ci . -t gcp_ci && \ + docker run -v $$(pwd):/go-control-plane $$(tty -s && echo "-it" || echo) gcp_ci /bin/bash -c /go-control-plane/build/do_benchmarks.sh + #-------------------------------------- #-- example xDS control plane server #-------------------------------------- diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000000..ddf8cb3ae1 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,65 @@ +# Benchmarks + +These benchmarks are designed to run off the integration test suite, and provide detailed profiling artifacts generated with pprof. + +## Prerequisites +- [Go 1.17+](https://golang.org/dl/) +- [Graphviz](https://graphviz.org/download/) +- [pprof](https://github.com/google/pprof) +- [Docker](https://www.docker.com/) + +## Running Locally + +To run the benchmarks locally, we take a similar apprach to the integration tests. + +### Requirements + +* Envoy binary `envoy` available: set `ENVOY` environment variable to the + location of the binary, or use the default value `/usr/local/bin/envoy` +* `go-control-plane` builds successfully +* Local installation of pprof and graphviz for profiler output + +### Steps + +To run the benchmark: +``` +make benchmark MODE=0 +``` + +There are 5 different modes all corresponding to various profiling outputs: +```go +const ( + PPROF_CPU int = iota + PPROF_HEAP + PPROF_MUTEX + PPROF_BLOCK + PPROF_GOROUTINE +) +``` +To specifiy the mode, use `MODE` environment variable that corresponds to the output you wish to evaluate. + +The output file will be stored at the project root with .pprof extension (e.g. `cpu.pprof`). + +Run `pprof` tool like so which will open up a shell. From there, you can run command such as `web` to visualize the result: + +```bash +go tool pprof cpu.pprof +(pprof) web +``` +For more information, run `help`. + +## Running With Docker + +To run the benchmarks, we just require the prerequisite of docker and go. + +### Steps + +To run the benchmarks: + +``` +make docker_benchmarks +``` + +This will generate all profile artifacts in the `./benchmarks/reports`. Graphical profile anaylsis is located in `/.benchmarks/pngs`. + +For more information on how to interpret these reports/graphs, [click here](https://github.com/google/pprof/blob/master/doc/README.md#graphical-reports) diff --git a/benchmarks/client/README.md b/benchmarks/client/README.md new file mode 100644 index 0000000000..3e39e0e22b --- /dev/null +++ b/benchmarks/client/README.md @@ -0,0 +1,3 @@ +# Benchmark Client + +This test client provides simulation of various workloads when communicating with the go-control-plane management server. diff --git a/benchmarks/client/main.go b/benchmarks/client/main.go new file mode 100755 index 0000000000..bb7f3a7564 --- /dev/null +++ b/benchmarks/client/main.go @@ -0,0 +1,48 @@ +package main + +import ( + "log" + "os" + "runtime" + "time" + + "github.com/urfave/cli/v2" + + "github.com/envoyproxy/go-control-plane/benchmarks/client/xds" +) + +func main() { + // Statically set the max procs + runtime.GOMAXPROCS(runtime.NumCPU()) + + app := &cli.App{ + Name: "xds-benchmark", + Usage: "xds benchmarking tool that simulates client workload", + Commands: []*cli.Command{ + { + Name: "run", + Aliases: []string{"r"}, + Usage: "run the benchmark with the provided duration", + Action: func(c *cli.Context) error { + arg := c.Args().First() + dur, err := time.ParseDuration(arg) + if err != nil { + return err + } + + sess, err := xds.NewSession("localhost:50000") + if err != nil { + return err + } + + return sess.Simulate(dur) + }, + }, + }, + } + + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} diff --git a/benchmarks/client/xds/opts.go b/benchmarks/client/xds/opts.go new file mode 100644 index 0000000000..f34a2015f7 --- /dev/null +++ b/benchmarks/client/xds/opts.go @@ -0,0 +1,48 @@ +package xds + +// Options are configuration settings for the discovery object +type Options struct { + NodeID string + Zone string + Cluster string + ResourceNames []string // List of Envoy resource names to subscribe to + ResourceType string // ex: type.googleapis.com/envoy.api.v2.ClusterLoadAssignment +} + +// Option follows the functional opts pattern +type Option func(*Options) + +// WithNode will inject the node id into the configuration object +func WithNode(id string) Option { + return func(o *Options) { + o.NodeID = id + } +} + +// WithZone will specificy which zone to use in the xDS discovery request +func WithZone(zone string) Option { + return func(o *Options) { + o.Zone = zone + } +} + +// WithCluster will specificy which cluster the request is announcing as +func WithCluster(cluster string) Option { + return func(o *Options) { + o.Cluster = cluster + } +} + +// WithResourceNames will inject a list of resources the user wants to place watches on +func WithResourceNames(names []string) Option { + return func(o *Options) { + o.ResourceNames = names + } +} + +// WithResourceType will inject the specific resource type that a user wants to stream +func WithResourceType(resource string) Option { + return func(o *Options) { + o.ResourceType = resource + } +} diff --git a/benchmarks/client/xds/xds.go b/benchmarks/client/xds/xds.go new file mode 100644 index 0000000000..ca1b7c6eb9 --- /dev/null +++ b/benchmarks/client/xds/xds.go @@ -0,0 +1,46 @@ +package xds + +import ( + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// Sess holds a grpc connection as well as config options to use during the simulation +type Sess struct { + Session *grpc.ClientConn + Opts Options +} + +// NewSession will dial a new benchmarking session with the configured options +func NewSession(url string, opts ...Option) (*Sess, error) { + var options Options + for _, o := range opts { + o(&options) + } + + conn, err := grpc.Dial(url, grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, err + } + + return &Sess{ + Session: conn, + Opts: options, + }, nil +} + +// Simulate will start an xDS stream which provides simulated clients communicating with an xDS server +func (s *Sess) Simulate(target time.Duration) error { + // Create a loop that will continually do work until the elapsed time as passed + for timeout := time.After(target); ; { + select { + case <-timeout: + return nil + default: + // TODO(alec): implement me + // Do some work + } + } +} diff --git a/benchmarks/client/xds/xds_test.go b/benchmarks/client/xds/xds_test.go new file mode 100644 index 0000000000..5b4b0d8c19 --- /dev/null +++ b/benchmarks/client/xds/xds_test.go @@ -0,0 +1 @@ +package xds diff --git a/build/do_benchmarks.sh b/build/do_benchmarks.sh new file mode 100755 index 0000000000..5257a3bad9 --- /dev/null +++ b/build/do_benchmarks.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -e +set -x + +go install golang.org/x/tools/cmd/goimports@latest +go get -u github.com/google/pprof +apt update && apt install -y graphviz + +cd /go-control-plane + +NUM_DIFF_LINES=`/go/bin/goimports -local github.com/envoyproxy/go-control-plane -d pkg | wc -l` +if [[ $NUM_DIFF_LINES > 0 ]]; then + echo "Failed format check. Run make format" + exit 1 +fi + +make benchmark MODE=0 +make benchmark MODE=1 +make benchmark MODE=2 +make benchmark MODE=3 +make benchmark MODE=4 + +mkdir -p benchmarks/reports +mkdir -p benchmarks/pngs + +# generate our consumable pprof profiles +pprof -text bin/test cpu.pprof > benchmarks/reports/cpu_text.txt +pprof -tree bin/test cpu.pprof > benchmarks/reports/cpu_tree.txt +pprof -traces bin/test cpu.pprof > benchmarks/reports/cpu_trace.txt + +pprof -text bin/test goroutine.pprof > benchmarks/reports/goroutine_text.txt +pprof -tree bin/test goroutine.pprof > benchmarks/reports/goroutine_tree.txt +pprof -traces bin/test goroutine.pprof > benchmarks/reports/goroutine_trace.txt + +pprof -text bin/test block.pprof > benchmarks/reports/block_text.txt +pprof -tree bin/test block.pprof > benchmarks/reports/block_tree.txt +pprof -traces bin/test block.pprof > benchmarks/reports/block_trace.txt + +pprof -text bin/test mutex.pprof > benchmarks/reports/mutex_text.txt +pprof -tree bin/test mutex.pprof > benchmarks/reports/mutex_tree.txt +pprof -traces bin/test mutex.pprof > benchmarks/reports/mutex_trace.txt + +pprof -text bin/test mem.pprof > benchmarks/reports/mem_text.txt +pprof -tree bin/test mem.pprof > benchmarks/reports/mem_tree.txt +pprof -traces bin/test mem.pprof > benchmarks/reports/mem_trace.txt + +# generate some cool pprof graphs +pprof -png bin/test cpu.pprof > benchmarks/pngs/cpu_graph.png +pprof -png bin/test goroutine.pprof > benchmarks/pngs/goroutine_graph.png +pprof -png bin/test block.pprof > benchmarks/pngs/block_graph.png +pprof -png bin/test mutex.pprof > benchmarks/pngs/mutex_graph.png +pprof -png bin/test mem.pprof > benchmarks/pngs/mem_graph.png diff --git a/build/integration.sh b/build/integration.sh index 811b547e03..a1e8184dfb 100755 --- a/build/integration.sh +++ b/build/integration.sh @@ -13,8 +13,8 @@ MESSAGE=$'Hi, there!\n' # Management server type. Valid values are "ads", "xds", "rest", "delta", or "delta-ads" XDS=${XDS:-ads} -# pprof profiler. True means turn on profiling -PPROF=${PPROF:-false} +# pprof profiler mode +MODE=${MODE:-0} # Number of RTDS layers. if [ "$XDS" = "ads" ]; then @@ -43,4 +43,4 @@ function cleanup() { trap cleanup EXIT # run the test suite (which also contains the control plane) -bin/test --xds=${XDS} --runtimes=${RUNTIMES} --pprof=${PPROF} -debug -message="$MESSAGE" "$@" +bin/test --xds=${XDS} --runtimes=${RUNTIMES} --pprofMode=${MODE} -debug -message="$MESSAGE" "$@" diff --git a/go.mod b/go.mod index 0c10b6d613..a16d3ee995 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,10 @@ require ( github.com/envoyproxy/protoc-gen-validate v0.9.1 github.com/golang/protobuf v1.5.2 github.com/google/go-cmp v0.5.9 + github.com/pkg/profile v1.7.0 github.com/prometheus/client_model v0.3.0 github.com/stretchr/testify v1.8.1 + github.com/urfave/cli/v2 v2.24.1 go.opentelemetry.io/proto/otlp v0.19.0 google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 google.golang.org/grpc v1.52.0 @@ -17,8 +19,13 @@ require ( ) require ( + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/fgprof v0.9.3 // indirect + github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/net v0.4.0 // indirect golang.org/x/sys v0.3.0 // indirect golang.org/x/text v0.5.0 // indirect diff --git a/go.sum b/go.sum index 4261e1ebdd..85a8e6da58 100644 --- a/go.sum +++ b/go.sum @@ -364,6 +364,7 @@ cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vf cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -387,6 +388,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc h1:PYXxkRUBGUMa5xgMVMDl62vEklZvKpVaxQeN9ie7Hfk= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -402,6 +405,8 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.9.1 h1:PS7VIOgmSVhWUEeZwTe7z7zouA22Cr590PzXKbZHOVY= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= +github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -476,6 +481,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -501,6 +508,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -513,6 +521,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= +github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -522,6 +532,8 @@ github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvq github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= @@ -536,6 +548,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= +github.com/urfave/cli/v2 v2.24.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -737,6 +753,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/test/main/README.md b/pkg/test/main/README.md index 61b9e5a5db..29c235a3f4 100644 --- a/pkg/test/main/README.md +++ b/pkg/test/main/README.md @@ -37,18 +37,3 @@ eventually converges to use the latest pushed configuration) for each run. You can run ```bin/test -help``` to get a list of the cli flags that the test program accepts. There are also comments in ```main.go```. - -## Using the pprof profiler - -One customization is to run the go language profiler [pprof](https://github.com/DataDog/go-profiler-notes/blob/main/pprof.md). See also . - -The profiler is normally off because it adds overhead to the tests. You can turn -it on with the command line option `--pprof`. There is an environment variable -`PPROF` for the `make` commands shown above. For example: - - (export PPROF=true; make integration.xds) - -The test will then write files of the form `block_profile_xds.pb.gz`. The files -get written to the root of the project, in the same place as the envoy logs. - -You can use `go tool pprof bin/test ` to analyze the profile data. diff --git a/pkg/test/main/main.go b/pkg/test/main/main.go index 02acc22637..4344303ef1 100644 --- a/pkg/test/main/main.go +++ b/pkg/test/main/main.go @@ -24,10 +24,10 @@ import ( "log" "net/http" "os" - "runtime" - "runtime/pprof" "time" + "github.com/pkg/profile" + "github.com/envoyproxy/go-control-plane/pkg/cache/v3" "github.com/envoyproxy/go-control-plane/pkg/server/v3" "github.com/envoyproxy/go-control-plane/pkg/test" @@ -35,6 +35,14 @@ import ( testv3 "github.com/envoyproxy/go-control-plane/pkg/test/v3" ) +const ( + PprofCPU int = iota + PprofHEAP + PprofMUTEX + PprofBLOCK + PprofGOROUTINE +) + var ( debug bool @@ -62,11 +70,11 @@ var ( nodeID string - pprofEnabled bool + pprofMode int ) func init() { - flag.BoolVar(&debug, "debug", false, "Use debug logging") + flag.BoolVar(&debug, "debug", false, "Use debug logging and enable pprof") // // These parameters control the ports that the integration test @@ -151,8 +159,8 @@ func init() { // These parameters control the the use of the pprof profiler // - // Enable use of the pprof profiler - flag.BoolVar(&pprofEnabled, "pprof", false, "Enable use of the pprof profiler") + // Pprof mode for which profiling method to use + flag.IntVar(&pprofMode, "pprofMode", 0, "Mode for pprof to run in: 0 -> CPU,1 -> heap, 2 -> mutex, 3 -> block, 4 -> goroutines") } @@ -161,13 +169,18 @@ func main() { flag.Parse() ctx := context.Background() - if pprofEnabled { - runtime.SetBlockProfileRate(1) - for _, prof := range []string{"block", "goroutine", "mutex"} { - log.Printf("turn on pprof %s profiler", prof) - if pprof.Lookup(prof) == nil { - pprof.NewProfile(prof) - } + if debug { + switch pprofMode { + case PprofCPU: + defer profile.Start(profile.CPUProfile, profile.ProfilePath("."), profile.NoShutdownHook).Stop() + case PprofHEAP: + defer profile.Start(profile.MemProfile, profile.ProfilePath(".")).Stop() + case PprofMUTEX: + defer profile.Start(profile.MutexProfile, profile.ProfilePath(".")).Stop() + case PprofBLOCK: + defer profile.Start(profile.BlockProfile, profile.ProfilePath(".")).Stop() + case PprofGOROUTINE: + defer profile.Start(profile.GoroutineProfile, profile.ProfilePath(".")).Stop() } } @@ -286,20 +299,6 @@ func main() { } } - if pprofEnabled { - for _, prof := range []string{"block", "goroutine", "mutex"} { - p := pprof.Lookup(prof) - filePath := fmt.Sprintf("%s_profile_%s.pb.gz", prof, mode) - log.Printf("storing %s profile for %s in %s", prof, mode, filePath) - f, err := os.Create(filePath) - if err != nil { - log.Fatalf("could not create %s profile %s: %s", prof, filePath, err) - } - p.WriteTo(f, 1) // nolint:errcheck - f.Close() - } - } - log.Printf("Test for %s passed!\n", mode) }