Adding OpenTelemtry Metrics

api/accounts/accounts.go

@@ -11,7 +11,7 @@ import (
 	"math/big"
 	"net/http"
 	"strconv"
-
+	
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/gorilla/mux"
@@ -349,10 +349,22 @@ func (a *Accounts) handleRevision(revision string) (*chain.BlockSummary, error)
 func (a *Accounts) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("/*").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(a.handleCallBatchCode))
-	sub.Path("/{address}").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(a.handleGetAccount))
-	sub.Path("/{address}/code").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(a.handleGetCode))
-	sub.Path("/{address}/storage/{key}").Methods("GET").HandlerFunc(utils.WrapHandlerFunc(a.handleGetStorage))
-	sub.Path("").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(a.handleCallContract))
-	sub.Path("/{address}").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(a.handleCallContract))
+	sub.Path("/*").
+		Methods("POST").
+		HandlerFunc(utils.MetricsWrapHandler("accounts_call_batch_code", utils.WrapHandlerFunc(a.handleCallBatchCode)))
+	sub.Path("/{address}").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandler("accounts_get_account", utils.WrapHandlerFunc(a.handleGetAccount)))
+	sub.Path("/{address}/code").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandler("accounts_get_code", utils.WrapHandlerFunc(a.handleGetCode)))
+	sub.Path("/{address}/storage/{key}").
+		Methods("GET").
+		HandlerFunc(utils.MetricsWrapHandler("accounts_get_storage", utils.WrapHandlerFunc(a.handleGetStorage)))
+	sub.Path("").
+		Methods("POST").
+		HandlerFunc(utils.MetricsWrapHandler("accounts_api_call_contract", utils.WrapHandlerFunc(a.handleCallContract)))
+	sub.Path("/{address}").
+		Methods("POST").
+		HandlerFunc(utils.MetricsWrapHandler("accounts_call_contract_address", utils.WrapHandlerFunc(a.handleCallContract)))
 }

api/api.go

@@ -24,6 +24,7 @@ import (
 	"github.com/vechain/thor/v2/chain"
 	"github.com/vechain/thor/v2/logdb"
 	"github.com/vechain/thor/v2/state"
+	"github.com/vechain/thor/v2/telemetry"
 	"github.com/vechain/thor/v2/thor"
 	"github.com/vechain/thor/v2/txpool"
 )
@@ -43,6 +44,7 @@ func New(
 	skipLogs bool,
 	allowCustomTracer bool,
 	forkConfig thor.ForkConfig,
+	enableMetrics bool,
 ) (http.HandlerFunc, func()) {
 	origins := strings.Split(strings.TrimSpace(allowedOrigins), ",")
 	for i, o := range origins {
@@ -51,6 +53,10 @@ func New(
 
 	router := mux.NewRouter()
 
+	if enableMetrics {
+		router.PathPrefix("/metrics").Handler(telemetry.Handler())
+	}
+
 	// to serve api doc and swagger-ui
 	router.PathPrefix("/doc").Handler(
 		http.StripPrefix("/doc/", http.FileServer(http.FS(doc.FS))),

api/blocks/blocks.go

@@ -136,5 +136,7 @@ func (b *Blocks) isTrunk(blkID thor.Bytes32, blkNum uint32) (bool, error) {
 
 func (b *Blocks) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
-	sub.Path("/{revision}").Methods("GET").HandlerFunc(utils.WrapHandlerFunc(b.handleGetBlock))
+	sub.Path("/{revision}").
+		Methods("GET").
+		HandlerFunc(utils.MetricsWrapHandler("blocks_get_block", utils.WrapHandlerFunc(b.handleGetBlock)))
 }

api/debug/debug.go

@@ -455,7 +455,13 @@ func (d *Debug) handleTraceCallOption(opt *TraceCallOption) (*xenv.TransactionCo
 func (d *Debug) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("/tracers").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(d.handleTraceClause))
-	sub.Path("/tracers/call").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(d.handleTraceCall))
-	sub.Path("/storage-range").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(d.handleDebugStorage))
+	sub.Path("/tracers").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandler("debug_trace_clause", utils.WrapHandlerFunc(d.handleTraceClause)))
+	sub.Path("/tracers/call").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandler("debug_trace_call", utils.WrapHandlerFunc(d.handleTraceCall)))
+	sub.Path("/storage-range").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandler("debug_debug_storage", utils.WrapHandlerFunc(d.handleDebugStorage)))
 }

api/events/events.go

@@ -61,5 +61,7 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error {
 func (e *Events) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(e.handleFilter))
+	sub.Path("").
+		Methods("POST").
+		HandlerFunc(utils.MetricsWrapHandler("events_filter", utils.WrapHandlerFunc(e.handleFilter)))
 }

api/node/node.go

@@ -33,5 +33,7 @@ func (n *Node) handleNetwork(w http.ResponseWriter, req *http.Request) error {
 func (n *Node) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("/network/peers").Methods("Get").HandlerFunc(utils.WrapHandlerFunc(n.handleNetwork))
+	sub.Path("/network/peers").
+		Methods("Get").
+		HandlerFunc(utils.MetricsWrapHandler("node_network", utils.WrapHandlerFunc(n.handleNetwork)))
 }

api/subscriptions/subscriptions.go

@@ -381,6 +381,10 @@ func (s *Subscriptions) Close() {
 func (s *Subscriptions) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("/txpool").Methods("Get").HandlerFunc(utils.WrapHandlerFunc(s.handlePendingTransactions))
-	sub.Path("/{subject}").Methods("Get").HandlerFunc(utils.WrapHandlerFunc(s.handleSubject))
+	sub.Path("/txpool").
+		Methods("Get").
+		HandlerFunc(utils.MetricsWrapHandler("subscriptions_pending_transactions", utils.WrapHandlerFunc(s.handlePendingTransactions)))
+	sub.Path("/{subject}").
+		Methods("Get").
+		HandlerFunc(utils.MetricsWrapHandler("subscriptions_subject", utils.WrapHandlerFunc(s.handleSubject)))
 }

api/transactions/transactions.go

@@ -215,7 +215,13 @@ func (t *Transactions) parseHead(head string) (thor.Bytes32, error) {
 func (t *Transactions) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(t.handleSendTransaction))
-	sub.Path("/{id}").Methods("GET").HandlerFunc(utils.WrapHandlerFunc(t.handleGetTransactionByID))
-	sub.Path("/{id}/receipt").Methods("GET").HandlerFunc(utils.WrapHandlerFunc(t.handleGetTransactionReceiptByID))
+	sub.Path("").
+		Methods("POST").
+		HandlerFunc(utils.MetricsWrapHandler("transactions_send_transaction", utils.WrapHandlerFunc(t.handleSendTransaction)))
+	sub.Path("/{id}").
+		Methods("GET").
+		HandlerFunc(utils.MetricsWrapHandler("transactions_get_transaction_by_id", utils.WrapHandlerFunc(t.handleGetTransactionByID)))
+	sub.Path("/{id}/receipt").
+		Methods("GET").
+		HandlerFunc(utils.MetricsWrapHandler("transactions_get_transaction_by_receipt", utils.WrapHandlerFunc(t.handleGetTransactionReceiptByID)))
 }

api/transfers/transfers.go

@@ -67,5 +67,7 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re
 func (t *Transfers) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("").Methods("POST").HandlerFunc(utils.WrapHandlerFunc(t.handleFilterTransferLogs))
+	sub.Path("").
+		Methods("POST").
+		HandlerFunc(utils.MetricsWrapHandler("transfers_transfer_logs", utils.WrapHandlerFunc(t.handleFilterTransferLogs)))
 }

api/utils/http.go

@@ -9,6 +9,9 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"time"
+
+	"github.com/vechain/thor/v2/telemetry"
 )
 
 type httpError struct {
@@ -67,6 +70,18 @@ func WrapHandlerFunc(f HandlerFunc) http.HandlerFunc {
 	}
 }
 
+// MetricsWrapHandler wraps a given handler and adds metrics to it
+func MetricsWrapHandler(endpoint string, f http.HandlerFunc) http.HandlerFunc {
+	counter := telemetry.Counter("api_" + endpoint + "_count_requests")
+	duration := telemetry.HistogramWithHTTPBuckets("api_" + endpoint + "_duration_ms")
+	return func(w http.ResponseWriter, r *http.Request) {
+		now := time.Now()
+		f(w, r)
+		counter.Add(1)
+		duration.Observe(time.Since(now).Milliseconds())
+	}
+}
+
 // content types
 const (
 	JSONContentType = "application/json; charset=utf-8"

cmd/thor/flags.go

@@ -142,4 +142,8 @@ var (
 		Value: 16,
 		Usage: "set tx limit per account in pool",
 	}
+	disableTelemetryFlag = cli.BoolFlag{
+		Name:  "disable-telemetry",
+		Usage: "disable telemetry server",
+	}
 )

cmd/thor/main.go

@@ -28,6 +28,7 @@ import (
 	"github.com/vechain/thor/v2/logdb"
 	"github.com/vechain/thor/v2/muxdb"
 	"github.com/vechain/thor/v2/state"
+	"github.com/vechain/thor/v2/telemetry"
 	"github.com/vechain/thor/v2/thor"
 	"github.com/vechain/thor/v2/txpool"
 	"gopkg.in/urfave/cli.v1"
@@ -86,6 +87,7 @@ func main() {
 			pprofFlag,
 			verifyLogsFlag,
 			disablePrunerFlag,
+			disableTelemetryFlag,
 		},
 		Action: defaultAction,
 		Commands: []cli.Command{
@@ -111,6 +113,7 @@ func main() {
 					txPoolLimitFlag,
 					txPoolLimitPerAccountFlag,
 					disablePrunerFlag,
+					disableTelemetryFlag,
 				},
 				Action: soloAction,
 			},
@@ -139,6 +142,12 @@ func defaultAction(ctx *cli.Context) error {
 	defer func() { log.Info("exited") }()
 
 	initLogger(ctx)
+	// enable telemetry as soon as possible
+	enableTelemetry := !ctx.Bool(disableTelemetryFlag.Name) // positive booleans are easier to read
+	if enableTelemetry {
+		telemetry.InitializeOtelTelemetry()
+	}
+
 	gene, forkConfig, err := selectGenesis(ctx)
 	if err != nil {
 		return err
@@ -207,7 +216,9 @@ func defaultAction(ctx *cli.Context) error {
 		ctx.Bool(pprofFlag.Name),
 		skipLogs,
 		ctx.Bool(apiAllowCustomTracerFlag.Name),
-		forkConfig)
+		forkConfig,
+		enableTelemetry,
+	)
 	defer func() { log.Info("closing API..."); apiCloser() }()
 
 	apiURL, srvCloser, err := startAPIServer(ctx, apiHandler, repo.GenesisBlock().Header().ID())
@@ -245,6 +256,13 @@ func soloAction(ctx *cli.Context) error {
 	defer func() { log.Info("exited") }()
 
 	initLogger(ctx)
+
+	// enable telemetry as soon as possible
+	enableTelemetry := !ctx.Bool(disableTelemetryFlag.Name) // positive booleans are easier to read
+	if enableTelemetry {
+		telemetry.InitializeOtelTelemetry()
+	}
+
 	gene := genesis.NewDevnet()
 	// Solo forks from the start
 	forkConfig := thor.ForkConfig{}
@@ -306,7 +324,9 @@ func soloAction(ctx *cli.Context) error {
 		ctx.Bool(pprofFlag.Name),
 		skipLogs,
 		ctx.Bool(apiAllowCustomTracerFlag.Name),
-		forkConfig)
+		forkConfig,
+		enableTelemetry,
+	)
 	defer func() { log.Info("closing API..."); apiCloser() }()
 
 	apiURL, srvCloser, err := startAPIServer(ctx, apiHandler, repo.GenesisBlock().Header().ID())

go.mod

@@ -21,10 +21,15 @@ require (
 	github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c
 	github.com/pkg/errors v0.8.0
 	github.com/pmezard/go-difflib v1.0.0
+	github.com/prometheus/client_golang v1.18.0
 	github.com/qianbin/directcache v0.9.7
-	github.com/stretchr/testify v1.7.2
+	github.com/stretchr/testify v1.8.4
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a
 	github.com/vechain/go-ecvrf v0.0.0-20220525125849-96fa0442e765
+	go.opentelemetry.io/otel v1.23.1
+	go.opentelemetry.io/otel/exporters/prometheus v0.45.2
+	go.opentelemetry.io/otel/metric v1.23.1
+	go.opentelemetry.io/otel/sdk/metric v1.23.1
 	golang.org/x/crypto v0.17.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gopkg.in/urfave/cli.v1 v1.20.0
@@ -33,13 +38,16 @@ require (
 
 require (
 	github.com/aristanetworks/goarista v0.0.0-20180222005525-c41ed3986faa // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 // indirect
 	github.com/cespare/cp v1.1.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/deckarep/golang-set v1.7.1 // indirect
 	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/go-stack/stack v1.7.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
@@ -49,10 +57,17 @@ require (
 	github.com/jackpal/go-nat-pmp v1.0.1 // indirect
 	github.com/mattn/go-colorable v0.0.9 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rjeczalik/notify v0.9.3 // indirect
+	go.opentelemetry.io/otel/sdk v1.23.1 // indirect
+	go.opentelemetry.io/otel/trace v1.23.1 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )