Skip to content

Commit

Permalink
Add otel-collector to collect benchmark metrics (#21)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lahsivjar
lahsivjar authored Aug 28, 2023
2 parents be79e9a + 3f01123 commit 035afa5
Show file tree
Hide file tree
Showing 13 changed files with 1,519 additions and 18 deletions.
6 changes: 4 additions & 2 deletions cmd/apmbench/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
## APMBench
# APMBench

APMBench allows running a set of benchmarks against APM supported endpoints.
APMBench allows running a set of benchmarks against APM supported endpoints. It
also allows collection of OTEL metrics from the servers implementing APM endpoints
and report it as benchmarking results.
4 changes: 3 additions & 1 deletion cmd/apmbench/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ var cfg struct {
Detailed bool
RunRE *regexp.Regexp
// Sorted list of agents count to be used for benchmarking
AgentsList []int
AgentsList []int
CollectorConfigYaml string
}

func init() {
Expand Down Expand Up @@ -58,4 +59,5 @@ func init() {
return nil
},
)
flag.StringVar(&cfg.CollectorConfigYaml, "collector-config-yaml", "", "configuration for otel collector")
}
72 changes: 71 additions & 1 deletion cmd/apmbench/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,85 @@
package main

import (
"context"
"errors"
"flag"
"log"
"sync"
"testing"
"time"

"github.com/elastic/apm-perf/internal/otelcollector"
"go.uber.org/zap"
)

func main() {
flag.Parse()

logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("failed to setup logger: %v", err)
}

extraMetrics := func(*testing.B) error { return nil }
resetFunc := func() { return }
if cfg.CollectorConfigYaml != "" {
logger.Info("starting otel collector...")

collectorCfg := otelcollector.DefaultConfig()
err := collectorCfg.LoadConfigFromYamlFile(cfg.CollectorConfigYaml)
if err != nil {
logger.Fatal("failed to load collector config", zap.Error(err))
}

collector, err := otelcollector.New(collectorCfg, logger)
if err != nil {
logger.Fatal("failed to create a new collector", zap.Error(err))
}
logger.Info("loaded collector configuration", zap.Object("config", &collectorCfg))

var wg sync.WaitGroup
defer wg.Wait()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

wg.Add(1)
go func() {
defer wg.Done()

if err := collector.Run(ctx); err != nil {
logger.Fatal("failed to run collector", zap.Error(err))
}
}()

extraMetrics = func(b *testing.B) error {
var errs []error
for _, cfg := range collectorCfg.InMemoryStoreConfig {
m, err := collector.GetAggregatedMetric(cfg)
if err != nil {
errs = append(errs, err)
continue
}
b.ReportMetric(m, cfg.Alias)
}
if len(errs) > 0 {
return errors.Join(errs...)
}
return nil
}
resetFunc = collector.Reset

// Wait for otel collector to be ready
ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := collector.Wait(ctx); err != nil {
logger.Fatal("failed to start collector", zap.Error(err))
}
}

if err := Run(
extraMetrics,
resetFunc,
Benchmark1000Transactions,
BenchmarkOTLPTraces,
BenchmarkAgentAll,
Expand All @@ -23,7 +93,7 @@ func main() {
BenchmarkAgentRuby,
Benchmark10000AggregationGroups,
); err != nil {
log.Fatal(err)
logger.Fatal("failed to run benchmarks", zap.Error(err))
}
}

Expand Down
16 changes: 13 additions & 3 deletions cmd/apmbench/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,11 @@ type result struct {
}

// Run runs all the given BenchmarkFunc.
func Run(fns ...BenchmarkFunc) error {
func Run(
extraMetrics func(*testing.B) error,
resetStore func(),
fns ...BenchmarkFunc,
) error {
type benchmark struct {
name string
fn BenchmarkFunc
Expand Down Expand Up @@ -70,7 +74,7 @@ func Run(fns ...BenchmarkFunc) error {
runtime.GOMAXPROCS(agents)
for _, b := range benchmarks {
name := fullBenchmarkName(b.name, agents)
result := runOne(b.fn)
result := runOne(extraMetrics, resetStore, b.fn)
// testing.Benchmark discards all output so the only thing we can
// retrive is the benchmark status and result.
if result.skipped {
Expand All @@ -87,7 +91,12 @@ func Run(fns ...BenchmarkFunc) error {
return nil
}

func runOne(fn BenchmarkFunc) (result result) {
func runOne(
extraMetrics func(*testing.B) error,
resetStore func(),
fn BenchmarkFunc,
) (result result) {
defer resetStore()
limiter := loadgen.GetNewLimiter(
loadgencfg.Config.EventRate.Burst,
loadgencfg.Config.EventRate.Interval,
Expand All @@ -106,6 +115,7 @@ func runOne(fn BenchmarkFunc) (result result) {

result.skipped = b.Skipped()
result.failed = b.Failed()
extraMetrics(b)
})
return result
}
Expand Down
74 changes: 71 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@ require (
go.elastic.co/apm/v2 v2.4.1
go.elastic.co/ecszap v1.0.1
go.elastic.co/fastjson v1.1.0
go.opentelemetry.io/collector v0.83.0
go.opentelemetry.io/collector/component v0.83.0
go.opentelemetry.io/collector/config/configtelemetry v0.83.0
go.opentelemetry.io/collector/consumer v0.83.0
go.opentelemetry.io/collector/exporter v0.83.0
go.opentelemetry.io/collector/pdata v1.0.0-rcv0014
go.opentelemetry.io/collector/receiver v0.83.0
go.opentelemetry.io/collector/receiver/otlpreceiver v0.83.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0
go.opentelemetry.io/otel/sdk v1.16.0
Expand All @@ -23,30 +31,89 @@ require (
)

require (
contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
github.com/armon/go-radix v1.0.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/elastic/go-licenser v0.4.0 // indirect
github.com/elastic/go-sysinfo v1.7.1 // indirect
github.com/elastic/go-windows v1.0.1 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jcchavezs/porto v0.1.0 // indirect
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/knadh/koanf v1.5.0 // indirect
github.com/knadh/koanf/v2 v2.0.1 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magefile/mage v1.9.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mostynb/go-grpc-compression v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/prometheus/statsd_exporter v0.22.7 // indirect
github.com/rs/cors v1.9.0 // indirect
github.com/shirou/gopsutil/v3 v3.23.7 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/collector/config/configauth v0.83.0 // indirect
go.opentelemetry.io/collector/config/configcompression v0.83.0 // indirect
go.opentelemetry.io/collector/config/configgrpc v0.83.0 // indirect
go.opentelemetry.io/collector/config/confighttp v0.83.0 // indirect
go.opentelemetry.io/collector/config/confignet v0.83.0 // indirect
go.opentelemetry.io/collector/config/configopaque v0.83.0 // indirect
go.opentelemetry.io/collector/config/configtls v0.83.0 // indirect
go.opentelemetry.io/collector/config/internal v0.83.0 // indirect
go.opentelemetry.io/collector/confmap v0.83.0 // indirect
go.opentelemetry.io/collector/connector v0.83.0 // indirect
go.opentelemetry.io/collector/extension v0.83.0 // indirect
go.opentelemetry.io/collector/extension/auth v0.83.0 // indirect
go.opentelemetry.io/collector/featuregate v1.0.0-rcv0014 // indirect
go.opentelemetry.io/collector/processor v0.83.0 // indirect
go.opentelemetry.io/collector/semconv v0.83.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.1-0.20230612162650-64be7e574a17 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.17.0 // indirect
go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/bridge/opencensus v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 // indirect
go.opentelemetry.io/otel/metric v1.16.0 // indirect
go.opentelemetry.io/otel/sdk/metric v0.39.0 // indirect
go.opentelemetry.io/otel/trace v1.16.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
Expand All @@ -55,10 +122,11 @@ require (
golang.org/x/sys v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/tools v0.7.0 // indirect
gonum.org/v1/gonum v0.13.0 // indirect
google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
howett.net/plist v1.0.0 // indirect
)
Loading

0 comments on commit 035afa5

Please sign in to comment.