Opentelemetry changes with optional tracing (#178)

* Opentelemetry changes with optional tracing

* Apply suggestions from code review

* Review fixes

* Add nil check

Co-authored-by: Joel Unzain <[email protected]>

WRPHandler.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"github.com/go-kit/kit/log/level"
+	"github.com/xmidt-org/bascule"
 	"net/http"
 
 	gokithttp "github.com/go-kit/kit/transport/http"
@@ -26,7 +28,7 @@ func withDeviceAccessCheck(errorLogger log.Logger, wrpRouterHandler wrphttp.Hand
 	}
 }
 
-func wrpRouterHandler(logger log.Logger, router device.Router) wrphttp.HandlerFunc {
+func wrpRouterHandler(logger log.Logger, router device.Router, ctxlogger func(ctx context.Context) bascule.Logger) wrphttp.HandlerFunc {
 	if logger == nil {
 		logger = logging.DefaultLogger()
 	}
@@ -35,14 +37,18 @@ func wrpRouterHandler(logger log.Logger, router device.Router) wrphttp.HandlerFu
 		panic("router is a required component")
 	}
 
-	errorLogger := logging.Error(logger)
-
 	return func(w wrphttp.ResponseWriter, r *wrphttp.Request) {
 		deviceRequest := &device.Request{
 			Message:  &r.Entity.Message,
 			Format:   r.Entity.Format,
 			Contents: r.Entity.Bytes,
 		}
+		var errorLogger log.Logger
+		if ctxlogger != nil && ctxlogger(r.Context()) != nil {
+			errorLogger = logging.Error(ctxlogger(r.Context()))
+		} else {
+			errorLogger = level.Error(logger)
+		}
 
 		// deviceRequest carries the context through the routing infrastructure
 		deviceResponse, err := router.Route(deviceRequest.WithContext(r.Context()))

WRPHandler_test.go

@@ -23,7 +23,7 @@ import (
 func testWRPHandlerNilRouter(t *testing.T) {
 	assert := assert.New(t)
 	assert.Panics(func() {
-		wrpRouterHandler(nil, nil)
+		wrpRouterHandler(nil, nil, nil)
 	})
 }
 
@@ -73,7 +73,7 @@ func testMessageHandlerServeHTTPDecodeError(t *testing.T) {
 		actualResponseBody map[string]interface{}
 
 		router  = new(mockRouter)
-		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(nil, router), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
+		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(nil, router, nil), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
 	)
 
 	handler.ServeHTTP(response, request)
@@ -122,7 +122,7 @@ func testMessageHandlerServeHTTPEncodeError(t *testing.T) {
 
 		router  = new(mockRouter)
 		d       = new(device.MockDevice)
-		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(nil, router), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
+		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(nil, router, nil), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
 
 		actualResponseBody     map[string]interface{}
 		expectedDeviceResponse = &device.Response{
@@ -174,7 +174,7 @@ func testMessageHandlerServeHTTPRouteError(t *testing.T, routeError error, expec
 		actualResponseBody map[string]interface{}
 
 		router  = new(mockRouter)
-		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(nil, router), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
+		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(nil, router, nil), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
 	)
 
 	router.On(
@@ -223,7 +223,7 @@ func testMessageHandlerServeHTTPEvent(t *testing.T, requestFormat wrp.Format) {
 		request  = httptest.NewRequest("POST", "/foo", bytes.NewReader(requestContents))
 
 		router  = new(mockRouter)
-		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(logging.NewTestLogger(nil, t), router), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
+		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(logging.NewTestLogger(nil, t), router, nil), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
 
 		actualDeviceRequest *device.Request
 	)
@@ -286,7 +286,7 @@ func testMessageHandlerServeHTTPRequestResponse(t *testing.T, responseFormat, re
 
 		router  = new(mockRouter)
 		d       = new(device.MockDevice)
-		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(logging.NewTestLogger(nil, t), router), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
+		handler = wrphttp.NewHTTPHandler(wrpRouterHandler(logging.NewTestLogger(nil, t), router, nil), wrphttp.WithDecoder(decorateRequestDecoder(wrphttp.DefaultDecoder())))
 
 		actualDeviceRequest    *device.Request
 		expectedDeviceResponse = &device.Response{

control.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"fmt"
+	"github.com/xmidt-org/candlelight"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux"
 	"net/http"
 
 	"github.com/go-kit/kit/log"
@@ -27,7 +29,7 @@ const (
 	drainPath  = "/device/drain"
 )
 
-func StartControlServer(logger log.Logger, manager device.Manager, deviceGate devicegate.Interface, registry xmetrics.Registry, v *viper.Viper) (func(http.Handler) http.Handler, error) {
+func StartControlServer(logger log.Logger, manager device.Manager, deviceGate devicegate.Interface, registry xmetrics.Registry, v *viper.Viper, tracing candlelight.Tracing) (func(http.Handler) http.Handler, error) {
 	if !v.IsSet(ControlKey) {
 		return xhttp.NilConstructor, nil
 	}
@@ -59,6 +61,13 @@ func StartControlServer(logger log.Logger, manager device.Manager, deviceGate de
 		apiHandler = r.PathPrefix(fmt.Sprintf("%s/%s", baseURI, version)).Subrouter()
 	)
 
+	otelMuxOptions := []otelmux.Option{
+		otelmux.WithPropagators(tracing.Propagator),
+		otelmux.WithTracerProvider(tracing.TracerProvider),
+	}
+
+	r.Use(otelmux.Middleware("control", otelMuxOptions...), candlelight.EchoFirstTraceNodeInfo(tracing.Propagator))
+
 	apiHandler.Handle(gatePath, &gate.Lever{Gate: g, Parameter: "open"}).Methods("POST", "PUT", "PATCH")
 
 	apiHandler.Handle(gatePath, &gate.Status{Gate: g}).Methods("GET")

deploy/packaging/talaria_spruce.yaml

@@ -490,3 +490,17 @@ service:
         # (Optional) defaults to empty struct
         # queryOptions:
         #   useCache: true
+
+# tracing provides configuration around traces using OpenTelemetry.
+# (Optional). By default, a 'noop' tracer provider is used and tracing is disabled.
+tracing:
+  # provider is the provider name. Currently, stdout, jaegar and zipkin are supported.
+  # 'noop' can also be used as provider to explicitly disable tracing.
+  provider: (( grab $TRACING_PROVIDER_NAME || "noop" ))
+
+  # skipTraceExport only applies when provider is stdout. Set skipTraceExport to true
+  # so that trace information is not written to stdout.
+  # skipTraceExport: true
+
+  # endpoint is where trace information should be routed. Applies to zipkin and jaegar.
+  endpoint: (( grab $TRACING_PROVIDER_ENDPOINT || "http://zipkin:9411/api/v2/spans" ))

go.mod

@@ -8,18 +8,22 @@ require (
 	github.com/fatih/structs v1.1.0
 	github.com/go-kit/kit v0.10.0
 	github.com/goph/emperror v0.17.3-0.20190703203600-60a8d9faa17b
-	github.com/gorilla/mux v1.7.4
+	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/schema v1.1.0 // indirect
 	github.com/justinas/alice v1.2.0
 	github.com/prometheus/client_golang v1.4.1
 	github.com/segmentio/ksuid v1.0.2
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/cast v1.3.1
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.7.0
-	github.com/stretchr/testify v1.5.1
+	github.com/spf13/viper v1.7.1
+	github.com/stretchr/testify v1.7.0
 	github.com/thedevsaddam/gojsonq/v2 v2.5.2
 	github.com/xmidt-org/bascule v0.9.0
+	github.com/xmidt-org/candlelight v0.0.4
 	github.com/xmidt-org/webpa-common v1.11.5-0.20210120235745-b98c7113262c
 	github.com/xmidt-org/wrp-go/v3 v3.0.1
+	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.19.0
+	go.opentelemetry.io/otel v0.19.0
+	go.opentelemetry.io/otel/trace v0.19.0
 )