From fae9af53e84180e6362256e544228bad55341013 Mon Sep 17 00:00:00 2001 From: otherview Date: Mon, 26 Feb 2024 15:25:00 +0000 Subject: [PATCH] Adding OpenTelemtry Metrics --- api/accounts/accounts.go | 26 +++++-- api/api.go | 6 ++ api/blocks/blocks.go | 4 +- api/debug/debug.go | 12 +++- api/events/events.go | 4 +- api/node/node.go | 4 +- api/subscriptions/subscriptions.go | 8 ++- api/transactions/transactions.go | 12 +++- api/transfers/transfers.go | 4 +- api/utils/http.go | 15 ++++ cmd/thor/flags.go | 4 ++ cmd/thor/main.go | 24 ++++++- go.mod | 21 +++++- go.sum | 46 ++++++++++-- telemetry/noop.go | 26 +++++++ telemetry/otel_prometheus.go | 109 +++++++++++++++++++++++++++++ telemetry/otel_prometheus_test.go | 53 ++++++++++++++ telemetry/telemetry.go | 42 +++++++++++ 18 files changed, 390 insertions(+), 30 deletions(-) create mode 100644 telemetry/noop.go create mode 100644 telemetry/otel_prometheus.go create mode 100644 telemetry/otel_prometheus_test.go create mode 100644 telemetry/telemetry.go diff --git a/api/accounts/accounts.go b/api/accounts/accounts.go index e4842c74d..1bd1959f6 100644 --- a/api/accounts/accounts.go +++ b/api/accounts/accounts.go @@ -11,7 +11,7 @@ import ( "math/big" "net/http" "strconv" - + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/gorilla/mux" @@ -349,10 +349,22 @@ func (a *Accounts) handleRevision(revision string) (*chain.BlockSummary, error) func (a *Accounts) Mount(root *mux.Router, pathPrefix string) { sub := root.PathPrefix(pathPrefix).Subrouter() - sub.Path("/*").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(a.handleCallBatchCode)) - sub.Path("/{address}").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(a.handleGetAccount)) - sub.Path("/{address}/code").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(a.handleGetCode)) - sub.Path("/{address}/storage/{key}").Methods("GET").HandlerFunc(utils.WrapHandlerFunc(a.handleGetStorage)) - sub.Path("").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(a.handleCallContract)) - sub.Path("/{address}").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(a.handleCallContract)) + sub.Path("/*"). + Methods("POST"). + HandlerFunc(utils.MetricsWrapHandler("accounts_call_batch_code", utils.WrapHandlerFunc(a.handleCallBatchCode))) + sub.Path("/{address}"). + Methods(http.MethodGet). + HandlerFunc(utils.MetricsWrapHandler("accounts_get_account", utils.WrapHandlerFunc(a.handleGetAccount))) + sub.Path("/{address}/code"). + Methods(http.MethodGet). + HandlerFunc(utils.MetricsWrapHandler("accounts_get_code", utils.WrapHandlerFunc(a.handleGetCode))) + sub.Path("/{address}/storage/{key}"). + Methods("GET"). + HandlerFunc(utils.MetricsWrapHandler("accounts_get_storage", utils.WrapHandlerFunc(a.handleGetStorage))) + sub.Path(""). + Methods("POST"). + HandlerFunc(utils.MetricsWrapHandler("accounts_api_call_contract", utils.WrapHandlerFunc(a.handleCallContract))) + sub.Path("/{address}"). + Methods("POST"). + HandlerFunc(utils.MetricsWrapHandler("accounts_call_contract_address", utils.WrapHandlerFunc(a.handleCallContract))) } diff --git a/api/api.go b/api/api.go index e59eb3462..6ae2ebf49 100644 --- a/api/api.go +++ b/api/api.go @@ -24,6 +24,7 @@ import ( "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/telemetry" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/txpool" ) @@ -43,6 +44,7 @@ func New( skipLogs bool, allowCustomTracer bool, forkConfig thor.ForkConfig, + enableMetrics bool, ) (http.HandlerFunc, func()) { origins := strings.Split(strings.TrimSpace(allowedOrigins), ",") for i, o := range origins { @@ -51,6 +53,10 @@ func New( router := mux.NewRouter() + if enableMetrics { + router.PathPrefix("/metrics").Handler(telemetry.Handler()) + } + // to serve api doc and swagger-ui router.PathPrefix("/doc").Handler( http.StripPrefix("/doc/", http.FileServer(http.FS(doc.FS))), diff --git a/api/blocks/blocks.go b/api/blocks/blocks.go index 280db69f6..a4422a7a0 100644 --- a/api/blocks/blocks.go +++ b/api/blocks/blocks.go @@ -136,5 +136,7 @@ func (b *Blocks) isTrunk(blkID thor.Bytes32, blkNum uint32) (bool, error) { func (b *Blocks) Mount(root *mux.Router, pathPrefix string) { sub := root.PathPrefix(pathPrefix).Subrouter() - sub.Path("/{revision}").Methods("GET").HandlerFunc(utils.WrapHandlerFunc(b.handleGetBlock)) + sub.Path("/{revision}"). + Methods("GET"). + HandlerFunc(utils.MetricsWrapHandler("blocks_get_block", utils.WrapHandlerFunc(b.handleGetBlock))) } diff --git a/api/debug/debug.go b/api/debug/debug.go index 9a7280e50..456e359f1 100644 --- a/api/debug/debug.go +++ b/api/debug/debug.go @@ -455,7 +455,13 @@ func (d *Debug) handleTraceCallOption(opt *TraceCallOption) (*xenv.TransactionCo func (d *Debug) Mount(root *mux.Router, pathPrefix string) { sub := root.PathPrefix(pathPrefix).Subrouter() - sub.Path("/tracers").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(d.handleTraceClause)) - sub.Path("/tracers/call").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(d.handleTraceCall)) - sub.Path("/storage-range").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(d.handleDebugStorage)) + sub.Path("/tracers"). + Methods(http.MethodPost). + HandlerFunc(utils.MetricsWrapHandler("debug_trace_clause", utils.WrapHandlerFunc(d.handleTraceClause))) + sub.Path("/tracers/call"). + Methods(http.MethodPost). + HandlerFunc(utils.MetricsWrapHandler("debug_trace_call", utils.WrapHandlerFunc(d.handleTraceCall))) + sub.Path("/storage-range"). + Methods(http.MethodPost). + HandlerFunc(utils.MetricsWrapHandler("debug_debug_storage", utils.WrapHandlerFunc(d.handleDebugStorage))) } diff --git a/api/events/events.go b/api/events/events.go index c5f5dddd5..10c89cd7d 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -61,5 +61,7 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error { func (e *Events) Mount(root *mux.Router, pathPrefix string) { sub := root.PathPrefix(pathPrefix).Subrouter() - sub.Path("").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(e.handleFilter)) + sub.Path(""). + Methods("POST"). + HandlerFunc(utils.MetricsWrapHandler("events_filter", utils.WrapHandlerFunc(e.handleFilter))) } diff --git a/api/node/node.go b/api/node/node.go index 6e685c7b9..96da6682c 100644 --- a/api/node/node.go +++ b/api/node/node.go @@ -33,5 +33,7 @@ func (n *Node) handleNetwork(w http.ResponseWriter, req *http.Request) error { func (n *Node) Mount(root *mux.Router, pathPrefix string) { sub := root.PathPrefix(pathPrefix).Subrouter() - sub.Path("/network/peers").Methods("Get").HandlerFunc(utils.WrapHandlerFunc(n.handleNetwork)) + sub.Path("/network/peers"). + Methods("Get"). + HandlerFunc(utils.MetricsWrapHandler("node_network", utils.WrapHandlerFunc(n.handleNetwork))) } diff --git a/api/subscriptions/subscriptions.go b/api/subscriptions/subscriptions.go index ab8cda0d6..92f883fcb 100644 --- a/api/subscriptions/subscriptions.go +++ b/api/subscriptions/subscriptions.go @@ -381,6 +381,10 @@ func (s *Subscriptions) Close() { func (s *Subscriptions) Mount(root *mux.Router, pathPrefix string) { sub := root.PathPrefix(pathPrefix).Subrouter() - sub.Path("/txpool").Methods("Get").HandlerFunc(utils.WrapHandlerFunc(s.handlePendingTransactions)) - sub.Path("/{subject}").Methods("Get").HandlerFunc(utils.WrapHandlerFunc(s.handleSubject)) + sub.Path("/txpool"). + Methods("Get"). + HandlerFunc(utils.MetricsWrapHandler("subscriptions_pending_transactions", utils.WrapHandlerFunc(s.handlePendingTransactions))) + sub.Path("/{subject}"). + Methods("Get"). + HandlerFunc(utils.MetricsWrapHandler("subscriptions_subject", utils.WrapHandlerFunc(s.handleSubject))) } diff --git a/api/transactions/transactions.go b/api/transactions/transactions.go index dc9acda75..71abfc5e7 100644 --- a/api/transactions/transactions.go +++ b/api/transactions/transactions.go @@ -215,7 +215,13 @@ func (t *Transactions) parseHead(head string) (thor.Bytes32, error) { func (t *Transactions) Mount(root *mux.Router, pathPrefix string) { sub := root.PathPrefix(pathPrefix).Subrouter() - sub.Path("").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(t.handleSendTransaction)) - sub.Path("/{id}").Methods("GET").HandlerFunc(utils.WrapHandlerFunc(t.handleGetTransactionByID)) - sub.Path("/{id}/receipt").Methods("GET").HandlerFunc(utils.WrapHandlerFunc(t.handleGetTransactionReceiptByID)) + sub.Path(""). + Methods("POST"). + HandlerFunc(utils.MetricsWrapHandler("transactions_send_transaction", utils.WrapHandlerFunc(t.handleSendTransaction))) + sub.Path("/{id}"). + Methods("GET"). + HandlerFunc(utils.MetricsWrapHandler("transactions_get_transaction_by_id", utils.WrapHandlerFunc(t.handleGetTransactionByID))) + sub.Path("/{id}/receipt"). + Methods("GET"). + HandlerFunc(utils.MetricsWrapHandler("transactions_get_transaction_by_receipt", utils.WrapHandlerFunc(t.handleGetTransactionReceiptByID))) } diff --git a/api/transfers/transfers.go b/api/transfers/transfers.go index bf4fb936f..d5c5d07d7 100644 --- a/api/transfers/transfers.go +++ b/api/transfers/transfers.go @@ -67,5 +67,7 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re func (t *Transfers) Mount(root *mux.Router, pathPrefix string) { sub := root.PathPrefix(pathPrefix).Subrouter() - sub.Path("").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(t.handleFilterTransferLogs)) + sub.Path(""). + Methods("POST"). + HandlerFunc(utils.MetricsWrapHandler("transfers_transfer_logs", utils.WrapHandlerFunc(t.handleFilterTransferLogs))) } diff --git a/api/utils/http.go b/api/utils/http.go index 652c3e408..a0fd7d4db 100644 --- a/api/utils/http.go +++ b/api/utils/http.go @@ -9,6 +9,9 @@ import ( "encoding/json" "io" "net/http" + "time" + + "github.com/vechain/thor/v2/telemetry" ) type httpError struct { @@ -67,6 +70,18 @@ func WrapHandlerFunc(f HandlerFunc) http.HandlerFunc { } } +// MetricsWrapHandler wraps a given handler and adds metrics to it +func MetricsWrapHandler(endpoint string, f http.HandlerFunc) http.HandlerFunc { + counter := telemetry.Counter("api_" + endpoint + "_count_requests") + duration := telemetry.HistogramWithHTTPBuckets("api_" + endpoint + "_duration_ms") + return func(w http.ResponseWriter, r *http.Request) { + now := time.Now() + f(w, r) + counter.Add(1) + duration.Observe(time.Since(now).Milliseconds()) + } +} + // content types const ( JSONContentType = "application/json; charset=utf-8" diff --git a/cmd/thor/flags.go b/cmd/thor/flags.go index 3f04d83a0..50f67e946 100644 --- a/cmd/thor/flags.go +++ b/cmd/thor/flags.go @@ -142,4 +142,8 @@ var ( Value: 16, Usage: "set tx limit per account in pool", } + disableTelemetryFlag = cli.BoolFlag{ + Name: "disable-telemetry", + Usage: "disable telemetry server", + } ) diff --git a/cmd/thor/main.go b/cmd/thor/main.go index 66118c9e7..c80a38996 100644 --- a/cmd/thor/main.go +++ b/cmd/thor/main.go @@ -28,6 +28,7 @@ import ( "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/telemetry" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/txpool" "gopkg.in/urfave/cli.v1" @@ -86,6 +87,7 @@ func main() { pprofFlag, verifyLogsFlag, disablePrunerFlag, + disableTelemetryFlag, }, Action: defaultAction, Commands: []cli.Command{ @@ -111,6 +113,7 @@ func main() { txPoolLimitFlag, txPoolLimitPerAccountFlag, disablePrunerFlag, + disableTelemetryFlag, }, Action: soloAction, }, @@ -139,6 +142,12 @@ func defaultAction(ctx *cli.Context) error { defer func() { log.Info("exited") }() initLogger(ctx) + // enable telemetry as soon as possible + enableTelemetry := !ctx.Bool(disableTelemetryFlag.Name) // positive booleans are easier to read + if enableTelemetry { + telemetry.InitializeOtelTelemetry() + } + gene, forkConfig, err := selectGenesis(ctx) if err != nil { return err @@ -207,7 +216,9 @@ func defaultAction(ctx *cli.Context) error { ctx.Bool(pprofFlag.Name), skipLogs, ctx.Bool(apiAllowCustomTracerFlag.Name), - forkConfig) + forkConfig, + enableTelemetry, + ) defer func() { log.Info("closing API..."); apiCloser() }() apiURL, srvCloser, err := startAPIServer(ctx, apiHandler, repo.GenesisBlock().Header().ID()) @@ -245,6 +256,13 @@ func soloAction(ctx *cli.Context) error { defer func() { log.Info("exited") }() initLogger(ctx) + + // enable telemetry as soon as possible + enableTelemetry := !ctx.Bool(disableTelemetryFlag.Name) // positive booleans are easier to read + if enableTelemetry { + telemetry.InitializeOtelTelemetry() + } + gene := genesis.NewDevnet() // Solo forks from the start forkConfig := thor.ForkConfig{} @@ -306,7 +324,9 @@ func soloAction(ctx *cli.Context) error { ctx.Bool(pprofFlag.Name), skipLogs, ctx.Bool(apiAllowCustomTracerFlag.Name), - forkConfig) + forkConfig, + enableTelemetry, + ) defer func() { log.Info("closing API..."); apiCloser() }() apiURL, srvCloser, err := startAPIServer(ctx, apiHandler, repo.GenesisBlock().Header().ID()) diff --git a/go.mod b/go.mod index e4d655aa4..c6477fa64 100644 --- a/go.mod +++ b/go.mod @@ -21,10 +21,15 @@ require ( github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c github.com/pkg/errors v0.8.0 github.com/pmezard/go-difflib v1.0.0 + github.com/prometheus/client_golang v1.18.0 github.com/qianbin/directcache v0.9.7 - github.com/stretchr/testify v1.7.2 + github.com/stretchr/testify v1.8.4 github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a github.com/vechain/go-ecvrf v0.0.0-20220525125849-96fa0442e765 + go.opentelemetry.io/otel v1.23.1 + go.opentelemetry.io/otel/exporters/prometheus v0.45.2 + go.opentelemetry.io/otel/metric v1.23.1 + go.opentelemetry.io/otel/sdk/metric v1.23.1 golang.org/x/crypto v0.17.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/urfave/cli.v1 v1.20.0 @@ -33,13 +38,16 @@ require ( require ( github.com/aristanetworks/goarista v0.0.0-20180222005525-c41ed3986faa // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 // indirect github.com/cespare/cp v1.1.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/deckarep/golang-set v1.7.1 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/fatih/color v1.7.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-stack/stack v1.7.0 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -49,10 +57,17 @@ require ( github.com/jackpal/go-nat-pmp v1.0.1 // indirect github.com/mattn/go-colorable v0.0.9 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/rjeczalik/notify v0.9.3 // indirect + go.opentelemetry.io/otel/sdk v1.23.1 // indirect + go.opentelemetry.io/otel/trace v1.23.1 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d6a7ede42..caa193e6b 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,15 @@ github.com/aristanetworks/goarista v0.0.0-20180222005525-c41ed3986faa h1:yCVE1EV github.com/aristanetworks/goarista v0.0.0-20180222005525-c41ed3986faa/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/beevik/ntp v0.2.0 h1:sGsd+kAXzT0bfVfzJfce04g+dSRfrs+tbQW8lweuYgw= github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -41,6 +44,11 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.7.0 h1:S04+lLfST9FvL8dl4R31wVUC/paZp/WQZbLmUgWboGw= @@ -61,6 +69,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= @@ -87,8 +96,8 @@ github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEM github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -103,6 +112,8 @@ github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbW github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a h1:8TGB3DFRNl06DB1Q6zBX+I7FDoCUZY2fmMS9WGUIIpw= github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -123,17 +134,26 @@ github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/qianbin/directcache v0.9.7 h1:DH6MdmU0fVjcKry57ju7U6akTFDBnLhHd0xOHZDq948= github.com/qianbin/directcache v0.9.7/go.mod h1:gZBpa9NqO1Qz7wZKO7t7atBA76bT8X0eM01PdveW4qc= github.com/rjeczalik/notify v0.9.3 h1:6rJAzHTGKXGj76sbRgDiDcYj/HniypXmSJo1SWakZeY= github.com/rjeczalik/notify v0.9.3/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/vechain/go-ecvrf v0.0.0-20220525125849-96fa0442e765 h1:jvr+TSivjObZmOKVdqlgeLtRhaDG27gE39PMuE2IJ24= github.com/vechain/go-ecvrf v0.0.0-20220525125849-96fa0442e765/go.mod h1:cwnTMgAVzMb30xMKnGI1LdU1NjMiPllYb7i3ibj/fzE= github.com/vechain/go-ethereum v1.8.15-0.20240130124343-9d419d1a61e5 h1:xvCKkXK/VDrHMh1E4oQ+1ctISCMwgHGW4rJ9nE+CsxA= @@ -142,6 +162,18 @@ github.com/vechain/goleveldb v1.0.1-0.20220809091043-51eb019c8655 h1:CbHcWpCi7wO github.com/vechain/goleveldb v1.0.1-0.20220809091043-51eb019c8655/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/otel v1.23.1 h1:Za4UzOqJYS+MUczKI320AtqZHZb7EqxO00jAHE0jmQY= +go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA= +go.opentelemetry.io/otel/exporters/prometheus v0.45.2 h1:pe2Jqk1K18As0RCw7J08QhgXNqr+6npx0a5W4IgAFA8= +go.opentelemetry.io/otel/exporters/prometheus v0.45.2/go.mod h1:B38pscHKI6bhFS44FDw0eFU3iqG3ASNIvY+fZgR5sAc= +go.opentelemetry.io/otel/metric v1.23.1 h1:PQJmqJ9u2QaJLBOELl1cxIdPcpbwzbkjfEyelTl2rlo= +go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI= +go.opentelemetry.io/otel/sdk v1.23.1 h1:O7JmZw0h76if63LQdsBMKQDWNb5oEcOThG9IrxscV+E= +go.opentelemetry.io/otel/sdk v1.23.1/go.mod h1:LzdEVR5am1uKOOwfBWFef2DCi1nu3SA8XQxx2IerWFk= +go.opentelemetry.io/otel/sdk/metric v1.23.1 h1:T9/8WsYg+ZqIpMWwdISVVrlGb/N0Jr1OHjR/alpKwzg= +go.opentelemetry.io/otel/sdk/metric v1.23.1/go.mod h1:8WX6WnNtHCgUruJ4TJ+UssQjMtpxkpX0zveQC8JG/E0= +go.opentelemetry.io/otel/trace v1.23.1 h1:4LrmmEd8AU2rFvU1zegmvqW7+kWarxtNOPyeL6HmYY8= +go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -185,8 +217,8 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -213,6 +245,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/telemetry/noop.go b/telemetry/noop.go new file mode 100644 index 000000000..91e9fa188 --- /dev/null +++ b/telemetry/noop.go @@ -0,0 +1,26 @@ +package telemetry + +import "net/http" + +// noopTelemetry implements a no operations telemetry service +type noopTelemetry struct{} + +func defaultNoopTelemetry() Telemetry { return &noopTelemetry{} } + +func (n *noopTelemetry) startServer() error { return nil } + +func (n *noopTelemetry) stopServer() error { return nil } + +func (n *noopTelemetry) GetOrCreateHistogramMeter(string, []int64) HistogramMeter { return &noopMetric } + +func (n *noopTelemetry) GetOrCreateCountMeter(string) CountMeter { return &noopMetric } + +func (n *noopTelemetry) GetOrCreateHandler() http.Handler { return nil } + +var noopMetric = noopMeters{} + +type noopMeters struct{} + +func (n noopMeters) Add(int64) {} + +func (n noopMeters) Observe(int64) {} diff --git a/telemetry/otel_prometheus.go b/telemetry/otel_prometheus.go new file mode 100644 index 000000000..d8e442a76 --- /dev/null +++ b/telemetry/otel_prometheus.go @@ -0,0 +1,109 @@ +package telemetry + +import ( + "context" + "log" + "net/http" + "sync" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/metric" + + metricsdk "go.opentelemetry.io/otel/sdk/metric" +) + +// InitializeOtelTelemetry creates a new instance of the OpenTelemetry service and +// sets the implementation as the default telemetry services +func InitializeOtelTelemetry() { + // don't allow for reset + if _, ok := telemetry.(*otelPrometheus); !ok { + telemetry = newOtelPrometheusTelemetry() + } +} + +type otelPrometheus struct { + meter metric.Meter + counters sync.Map + histograms sync.Map +} + +func newOtelPrometheusTelemetry() Telemetry { + exporter, err := prometheus.New() + if err != nil { + log.Fatal(err) + } + provider := metricsdk.NewMeterProvider(metricsdk.WithReader(exporter)) + + return &otelPrometheus{ + meter: provider.Meter("node_telemetry"), + counters: sync.Map{}, + histograms: sync.Map{}, + } +} + +func (o *otelPrometheus) GetOrCreateCountMeter(name string) CountMeter { + var meter CountMeter + mapItem, ok := o.counters.Load(name) + if !ok { + meter = o.newCountMeter(name) + o.counters.Store(name, meter) + } else { + meter = mapItem.(CountMeter) + } + return meter +} + +func (o *otelPrometheus) GetOrCreateHandler() http.Handler { + return promhttp.Handler() +} + +func (o *otelPrometheus) GetOrCreateHistogramMeter(name string, buckets []int64) HistogramMeter { + var meter HistogramMeter + mapItem, ok := o.histograms.Load(name) + if !ok { + meter = o.newHistogramMeter(name, buckets) + o.histograms.Store(name, meter) + } else { + meter = mapItem.(HistogramMeter) + } + return meter +} + +func (o *otelPrometheus) newHistogramMeter(name string, buckets []int64) HistogramMeter { + var floatBuckets []float64 + for _, bucket := range buckets { + floatBuckets = append(floatBuckets, float64(bucket)) + } + // purposefully ignoring the error given its strict properties + hist, _ := o.meter.Int64Histogram(name, metric.WithExplicitBucketBoundaries(floatBuckets...)) + + return &otelHistogramMeter{ + histogram: hist, + } +} + +type otelHistogramMeter struct { + histogram metric.Int64Histogram +} + +func (c *otelHistogramMeter) Observe(i int64) { + c.histogram.Record(context.Background(), i) +} + +func (o *otelPrometheus) newCountMeter(name string) CountMeter { + // purposefully ignoring the error given only the name is supplied + counter, _ := o.meter.Int64Counter(name) + + return &otelCountMeter{ + counter: counter, + } +} + +type otelCountMeter struct { + counter metric.Int64Counter +} + +func (c *otelCountMeter) Add(i int64) { + c.counter.Add(context.Background(), i) +} diff --git a/telemetry/otel_prometheus_test.go b/telemetry/otel_prometheus_test.go new file mode 100644 index 000000000..630410456 --- /dev/null +++ b/telemetry/otel_prometheus_test.go @@ -0,0 +1,53 @@ +package telemetry + +import ( + "math/rand" + "net/http" + "net/http/httptest" + "testing" + + "github.com/prometheus/common/expfmt" + "github.com/stretchr/testify/require" +) + +func TestOtelPromTelemetry(t *testing.T) { + InitializeOtelTelemetry() + server := httptest.NewServer(Handler()) + + t.Cleanup(func() { + server.Close() + }) + + // 2 ways of accessing it - useful to avoid lookups + count1 := Counter("count1") + Counter("count2") + + count1.Add(1) + randCount2 := rand.Intn(100) + 1 + for i := 0; i < randCount2; i++ { + Counter("count2").Add(1) + } + + hist := Histogram("hist1") + histTotal := 0 + for i := 0; i < rand.Intn(100)+1; i++ { + hist.Observe(int64(i)) + histTotal += i + } + + // Make a request to the metrics endpoint + resp, err := http.Get(server.URL + "/metrics") + if err != nil { + t.Errorf("Failed to make GET request: %v", err) + } + + defer resp.Body.Close() + + parser := expfmt.TextParser{} + metrics, err := parser.TextToMetricFamilies(resp.Body) + require.NoError(t, err) + + require.Equal(t, metrics["count1_total"].GetMetric()[0].GetCounter().GetValue(), float64(1)) + require.Equal(t, metrics["count2_total"].GetMetric()[0].GetCounter().GetValue(), float64(randCount2)) + require.Equal(t, metrics["hist1"].GetMetric()[0].GetHistogram().GetSampleSum(), float64(histTotal)) +} diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go new file mode 100644 index 000000000..4e1275d3e --- /dev/null +++ b/telemetry/telemetry.go @@ -0,0 +1,42 @@ +package telemetry + +import "net/http" + +// telemetry is a singleton service that provides global access to a set of meters +// it wraps multiple implementations and defaults to a no-op implementation +var telemetry = defaultNoopTelemetry() // defaults to a Noop implementation of the telemetry service + +// Telemetry defines the interface for telemetry service implementations +type Telemetry interface { + GetOrCreateCountMeter(name string) CountMeter + GetOrCreateHistogramMeter(name string, buckets []int64) HistogramMeter + GetOrCreateHandler() http.Handler +} + +// Handler returns the http handler for retrieving metrics +func Handler() http.Handler { + return telemetry.GetOrCreateHandler() +} + +// HistogramMeter represents the type of metric that is calculated by aggregating +// as a Histogram of all reported measurements over a time interval. +type HistogramMeter interface { + Observe(int64) +} + +func Histogram(name string) HistogramMeter { + return telemetry.GetOrCreateHistogramMeter(name, nil) +} +func HistogramWithHTTPBuckets(name string) HistogramMeter { + return telemetry.GetOrCreateHistogramMeter(name, defaultHTTPBuckets) +} + +var defaultHTTPBuckets = []int64{0, 150, 300, 450, 600, 900, 1200, 1500, 3000} + +// CountMeter is a cumulative metric that represents a single monotonically increasing counter +// whose value can only increase or be reset to zero on restart. +type CountMeter interface { + Add(int64) +} + +func Counter(name string) CountMeter { return telemetry.GetOrCreateCountMeter(name) }