diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go index e8d836869d..5efa43149a 100644 --- a/ee/query-service/app/api/api.go +++ b/ee/query-service/app/api/api.go @@ -38,9 +38,10 @@ type APIHandlerOptions struct { Cache cache.Cache Gateway *httputil.ReverseProxy // Querier Influx Interval - FluxInterval time.Duration - UseLogsNewSchema bool - UseLicensesV3 bool + FluxInterval time.Duration + UseLogsNewSchema bool + UseTraceNewSchema bool + UseLicensesV3 bool } type APIHandler struct { @@ -66,6 +67,7 @@ func NewAPIHandler(opts APIHandlerOptions) (*APIHandler, error) { Cache: opts.Cache, FluxInterval: opts.FluxInterval, UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, UseLicensesV3: opts.UseLicensesV3, }) diff --git a/ee/query-service/app/api/traces.go b/ee/query-service/app/api/traces.go index 3864fc672e..5c65089cd0 100644 --- a/ee/query-service/app/api/traces.go +++ b/ee/query-service/app/api/traces.go @@ -2,32 +2,31 @@ package api import ( "net/http" - - "go.signoz.io/signoz/ee/query-service/app/db" - "go.signoz.io/signoz/ee/query-service/model" - baseapp "go.signoz.io/signoz/pkg/query-service/app" - basemodel "go.signoz.io/signoz/pkg/query-service/model" - "go.uber.org/zap" ) func (ah *APIHandler) searchTraces(w http.ResponseWriter, r *http.Request) { - if !ah.CheckFeature(basemodel.SmartTraceDetail) { - zap.L().Info("SmartTraceDetail feature is not enabled in this plan") - ah.APIHandler.SearchTraces(w, r) - return - } - searchTracesParams, err := baseapp.ParseSearchTracesParams(r) - if err != nil { - RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params") - return - } - - result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm) - if ah.HandleError(w, err, http.StatusBadRequest) { - return - } - - ah.WriteJSON(w, r, result) + ah.APIHandler.SearchTraces(w, r) + return + + // This is commented since this will be taken care by new trace API + + // if !ah.CheckFeature(basemodel.SmartTraceDetail) { + // zap.L().Info("SmartTraceDetail feature is not enabled in this plan") + // ah.APIHandler.SearchTraces(w, r) + // return + // } + // searchTracesParams, err := baseapp.ParseSearchTracesParams(r) + // if err != nil { + // RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params") + // return + // } + + // result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm) + // if ah.HandleError(w, err, http.StatusBadRequest) { + // return + // } + + // ah.WriteJSON(w, r, result) } diff --git a/ee/query-service/app/db/reader.go b/ee/query-service/app/db/reader.go index fcab1cb991..9794abd013 100644 --- a/ee/query-service/app/db/reader.go +++ b/ee/query-service/app/db/reader.go @@ -26,8 +26,9 @@ func NewDataConnector( dialTimeout time.Duration, cluster string, useLogsNewSchema bool, + useTraceNewSchema bool, ) *ClickhouseReader { - ch := basechr.NewReader(localDB, promConfigPath, lm, maxIdleConns, maxOpenConns, dialTimeout, cluster, useLogsNewSchema) + ch := basechr.NewReader(localDB, promConfigPath, lm, maxIdleConns, maxOpenConns, dialTimeout, cluster, useLogsNewSchema, useTraceNewSchema) return &ClickhouseReader{ conn: ch.GetConn(), appdb: localDB, diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index a8acbc46e9..b58a90b9c4 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -77,6 +77,7 @@ type ServerOptions struct { Cluster string GatewayUrl string UseLogsNewSchema bool + UseTraceNewSchema bool UseLicensesV3 bool } @@ -156,6 +157,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.DialTimeout, serverOptions.Cluster, serverOptions.UseLogsNewSchema, + serverOptions.UseTraceNewSchema, ) go qb.Start(readerReady) reader = qb @@ -189,6 +191,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.DisableRules, lm, serverOptions.UseLogsNewSchema, + serverOptions.UseTraceNewSchema, ) if err != nil { @@ -270,6 +273,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { FluxInterval: fluxInterval, Gateway: gatewayProxy, UseLogsNewSchema: serverOptions.UseLogsNewSchema, + UseTraceNewSchema: serverOptions.UseTraceNewSchema, UseLicensesV3: serverOptions.UseLicensesV3, } @@ -737,7 +741,8 @@ func makeRulesManager( cache cache.Cache, disableRules bool, fm baseint.FeatureLookup, - useLogsNewSchema bool) (*baserules.Manager, error) { + useLogsNewSchema bool, + useTraceNewSchema bool) (*baserules.Manager, error) { // create engine pqle, err := pqle.FromConfigPath(promConfigPath) @@ -767,8 +772,9 @@ func makeRulesManager( EvalDelay: baseconst.GetEvalDelay(), PrepareTaskFunc: rules.PrepareTaskFunc, - PrepareTestRuleFunc: rules.TestNotification, UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, + PrepareTestRuleFunc: rules.TestNotification, } // create Manager diff --git a/ee/query-service/main.go b/ee/query-service/main.go index 55e70893e6..23824bd636 100644 --- a/ee/query-service/main.go +++ b/ee/query-service/main.go @@ -94,6 +94,7 @@ func main() { var cluster string var useLogsNewSchema bool + var useTraceNewSchema bool var useLicensesV3 bool var cacheConfigPath, fluxInterval string var enableQueryServiceLogOTLPExport bool @@ -105,6 +106,7 @@ func main() { var gatewayUrl string flag.BoolVar(&useLogsNewSchema, "use-logs-new-schema", false, "use logs_v2 schema for logs") + flag.BoolVar(&useTraceNewSchema, "use-trace-new-schema", false, "use new schema for traces") flag.BoolVar(&useLicensesV3, "use-licenses-v3", false, "use licenses_v3 schema for licenses") flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") flag.StringVar(&skipTopLvlOpsPath, "skip-top-level-ops", "", "(config file to skip top level operations)") @@ -145,6 +147,7 @@ func main() { Cluster: cluster, GatewayUrl: gatewayUrl, UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, UseLicensesV3: useLicensesV3, } diff --git a/ee/query-service/rules/manager.go b/ee/query-service/rules/manager.go index 9843d108d8..00e0882f36 100644 --- a/ee/query-service/rules/manager.go +++ b/ee/query-service/rules/manager.go @@ -26,6 +26,7 @@ func PrepareTaskFunc(opts baserules.PrepareTaskOptions) (baserules.Task, error) opts.FF, opts.Reader, opts.UseLogsNewSchema, + opts.UseTraceNewSchema, baserules.WithEvalDelay(opts.ManagerOpts.EvalDelay), ) @@ -122,6 +123,7 @@ func TestNotification(opts baserules.PrepareTestRuleOptions) (int, *basemodel.Ap opts.FF, opts.Reader, opts.UseLogsNewSchema, + opts.UseTraceNewSchema, baserules.WithSendAlways(), baserules.WithSendUnmatched(), ) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 765349bee0..349cce4aef 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -167,7 +167,7 @@ func NewReader( dialTimeout time.Duration, cluster string, useLogsNewSchema bool, - // useTraceNewSchema bool, // TODO: uncomment this in integration PR + useTraceNewSchema bool, ) *ClickHouseReader { datasource := os.Getenv("ClickHouseUrl") @@ -178,7 +178,7 @@ func NewReader( zap.L().Fatal("failed to initialize ClickHouse", zap.Error(err)) } - return NewReaderFromClickhouseConnection(db, options, localDB, configFile, featureFlag, cluster, useLogsNewSchema) + return NewReaderFromClickhouseConnection(db, options, localDB, configFile, featureFlag, cluster, useLogsNewSchema, useTraceNewSchema) } func NewReaderFromClickhouseConnection( @@ -189,7 +189,7 @@ func NewReaderFromClickhouseConnection( featureFlag interfaces.FeatureLookup, cluster string, useLogsNewSchema bool, - // useTraceNewSchema bool, + useTraceNewSchema bool, ) *ClickHouseReader { alertManager, err := am.New() if err != nil { @@ -229,11 +229,10 @@ func NewReaderFromClickhouseConnection( traceTableName := options.primary.IndexTable traceLocalTableName := options.primary.LocalIndexTable - // TODO: uncomment this in integration PR - // if useTraceNewSchema { - // traceTableName = options.primary.TraceIndexTableV3 - // traceLocalTableName = options.primary.TraceLocalTableNameV3 - // } + if useTraceNewSchema { + traceTableName = options.primary.TraceIndexTableV3 + traceLocalTableName = options.primary.TraceLocalTableNameV3 + } return &ClickHouseReader{ db: wrap, @@ -262,7 +261,8 @@ func NewReaderFromClickhouseConnection( cluster: cluster, queryProgressTracker: queryprogress.NewQueryProgressTracker(), - useLogsNewSchema: useLogsNewSchema, + useLogsNewSchema: useLogsNewSchema, + useTraceNewSchema: useTraceNewSchema, logsTableV2: options.primary.LogsTableV2, logsLocalTableV2: options.primary.LogsLocalTableV2, @@ -271,7 +271,6 @@ func NewReaderFromClickhouseConnection( logsTableName: logsTableName, logsLocalTableName: logsLocalTableName, - // useTraceNewSchema: useTraceNewSchema, traceLocalTableName: traceLocalTableName, traceTableName: traceTableName, traceResourceTableV3: options.primary.TraceResourceTableV3, diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 5fb604d7a9..5031cf7123 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -39,6 +39,7 @@ import ( querierV2 "go.signoz.io/signoz/pkg/query-service/app/querier/v2" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/cache" "go.signoz.io/signoz/pkg/query-service/common" @@ -110,8 +111,9 @@ type APIHandler struct { // Websocket connection upgrader Upgrader *websocket.Upgrader - UseLogsNewSchema bool - UseLicensesV3 bool + UseLogsNewSchema bool + UseTraceNewSchema bool + UseLicensesV3 bool hostsRepo *inframetrics.HostsRepo processesRepo *inframetrics.ProcessesRepo @@ -163,6 +165,7 @@ type APIHandlerOpts struct { // Use Logs New schema UseLogsNewSchema bool + UseTraceNewSchema bool // Use Licenses V3 structure UseLicensesV3 bool } @@ -176,21 +179,23 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { } querierOpts := querier.QuerierOptions{ - Reader: opts.Reader, - Cache: opts.Cache, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FluxInterval: opts.FluxInterval, - FeatureLookup: opts.FeatureFlags, - UseLogsNewSchema: opts.UseLogsNewSchema, + Reader: opts.Reader, + Cache: opts.Cache, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FluxInterval: opts.FluxInterval, + FeatureLookup: opts.FeatureFlags, + UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, } querierOptsV2 := querierV2.QuerierOptions{ - Reader: opts.Reader, - Cache: opts.Cache, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FluxInterval: opts.FluxInterval, - FeatureLookup: opts.FeatureFlags, - UseLogsNewSchema: opts.UseLogsNewSchema, + Reader: opts.Reader, + Cache: opts.Cache, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FluxInterval: opts.FluxInterval, + FeatureLookup: opts.FeatureFlags, + UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, } querier := querier.NewQuerier(querierOpts) @@ -224,6 +229,7 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { querier: querier, querierV2: querierv2, UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, UseLicensesV3: opts.UseLicensesV3, hostsRepo: hostsRepo, processesRepo: processesRepo, @@ -242,9 +248,14 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { logsQueryBuilder = logsv4.PrepareLogsQuery } + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if opts.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } + builderOpts := queryBuilder.QueryBuilderOptions{ BuildMetricQuery: metricsv3.PrepareMetricQuery, - BuildTraceQuery: tracesV3.PrepareTracesQuery, + BuildTraceQuery: tracesQueryBuilder, BuildLogQuery: logsQueryBuilder, } aH.queryBuilder = queryBuilder.NewQueryBuilder(builderOpts, aH.featureFlags) @@ -4330,7 +4341,12 @@ func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.Que RespondError(w, apiErrObj, errQuriesByName) return } - tracesV3.Enrich(queryRangeParams, spanKeys) + if aH.UseTraceNewSchema { + tracesV4.Enrich(queryRangeParams, spanKeys) + } else { + tracesV3.Enrich(queryRangeParams, spanKeys) + } + } // WARN: Only works for AND operator in traces query @@ -4800,7 +4816,11 @@ func (aH *APIHandler) queryRangeV4(ctx context.Context, queryRangeParams *v3.Que RespondError(w, apiErrObj, errQuriesByName) return } - tracesV3.Enrich(queryRangeParams, spanKeys) + if aH.UseTraceNewSchema { + tracesV4.Enrich(queryRangeParams, spanKeys) + } else { + tracesV3.Enrich(queryRangeParams, spanKeys) + } } // WARN: Only works for AND operator in traces query diff --git a/pkg/query-service/app/querier/helper.go b/pkg/query-service/app/querier/helper.go index 798eb8f0b7..1b2acbab8b 100644 --- a/pkg/query-service/app/querier/helper.go +++ b/pkg/query-service/app/querier/helper.go @@ -10,6 +10,7 @@ import ( logsV4 "go.signoz.io/signoz/pkg/query-service/app/logs/v4" metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/common" "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" @@ -158,11 +159,16 @@ func (q *querier) runBuilderQuery( if builderQuery.DataSource == v3.DataSourceTraces { + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if q.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } + var query string var err error // for ts query with group by and limit form two queries if params.CompositeQuery.PanelType == v3.PanelTypeGraph && builderQuery.Limit > 0 && len(builderQuery.GroupBy) > 0 { - limitQuery, err := tracesV3.PrepareTracesQuery( + limitQuery, err := tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, @@ -173,7 +179,7 @@ func (q *querier) runBuilderQuery( ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} return } - placeholderQuery, err := tracesV3.PrepareTracesQuery( + placeholderQuery, err := tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, @@ -186,7 +192,7 @@ func (q *querier) runBuilderQuery( } query = fmt.Sprintf(placeholderQuery, limitQuery) } else { - query, err = tracesV3.PrepareTracesQuery( + query, err = tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index fc9b0d9431..9ef0e2582c 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -11,6 +11,7 @@ import ( metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/common" "go.signoz.io/signoz/pkg/query-service/constants" chErrors "go.signoz.io/signoz/pkg/query-service/errors" @@ -65,10 +66,11 @@ type QuerierOptions struct { FeatureLookup interfaces.FeatureLookup // used for testing - TestingMode bool - ReturnedSeries []*v3.Series - ReturnedErr error - UseLogsNewSchema bool + TestingMode bool + ReturnedSeries []*v3.Series + ReturnedErr error + UseLogsNewSchema bool + UseTraceNewSchema bool } func NewQuerier(opts QuerierOptions) interfaces.Querier { @@ -76,6 +78,10 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { if opts.UseLogsNewSchema { logsQueryBuilder = logsV4.PrepareLogsQuery } + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if opts.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } qc := querycache.NewQueryCache(querycache.WithCache(opts.Cache), querycache.WithFluxInterval(opts.FluxInterval)) @@ -87,16 +93,17 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { fluxInterval: opts.FluxInterval, builder: queryBuilder.NewQueryBuilder(queryBuilder.QueryBuilderOptions{ - BuildTraceQuery: tracesV3.PrepareTracesQuery, + BuildTraceQuery: tracesQueryBuilder, BuildLogQuery: logsQueryBuilder, BuildMetricQuery: metricsV3.PrepareMetricQuery, }, opts.FeatureLookup), featureLookUp: opts.FeatureLookup, - testingMode: opts.TestingMode, - returnedSeries: opts.ReturnedSeries, - returnedErr: opts.ReturnedErr, - UseLogsNewSchema: opts.UseLogsNewSchema, + testingMode: opts.TestingMode, + returnedSeries: opts.ReturnedSeries, + returnedErr: opts.ReturnedErr, + UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, } } diff --git a/pkg/query-service/app/querier/querier_test.go b/pkg/query-service/app/querier/querier_test.go index 62e7e97c8a..fcaf13624f 100644 --- a/pkg/query-service/app/querier/querier_test.go +++ b/pkg/query-service/app/querier/querier_test.go @@ -1383,6 +1383,7 @@ func Test_querier_runWindowBasedListQuery(t *testing.T) { featureManager.StartManager(), "", true, + true, ) q := &querier{ diff --git a/pkg/query-service/app/querier/v2/helper.go b/pkg/query-service/app/querier/v2/helper.go index 09d6cc2309..b62fffe106 100644 --- a/pkg/query-service/app/querier/v2/helper.go +++ b/pkg/query-service/app/querier/v2/helper.go @@ -11,6 +11,7 @@ import ( metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" metricsV4 "go.signoz.io/signoz/pkg/query-service/app/metrics/v4" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/common" "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" @@ -158,11 +159,16 @@ func (q *querier) runBuilderQuery( if builderQuery.DataSource == v3.DataSourceTraces { + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if q.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } + var query string var err error // for ts query with group by and limit form two queries if params.CompositeQuery.PanelType == v3.PanelTypeGraph && builderQuery.Limit > 0 && len(builderQuery.GroupBy) > 0 { - limitQuery, err := tracesV3.PrepareTracesQuery( + limitQuery, err := tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, @@ -173,7 +179,7 @@ func (q *querier) runBuilderQuery( ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} return } - placeholderQuery, err := tracesV3.PrepareTracesQuery( + placeholderQuery, err := tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, @@ -186,7 +192,7 @@ func (q *querier) runBuilderQuery( } query = fmt.Sprintf(placeholderQuery, limitQuery) } else { - query, err = tracesV3.PrepareTracesQuery( + query, err = tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 883b1055f4..86a5af334a 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -11,6 +11,7 @@ import ( metricsV4 "go.signoz.io/signoz/pkg/query-service/app/metrics/v4" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/common" "go.signoz.io/signoz/pkg/query-service/constants" chErrors "go.signoz.io/signoz/pkg/query-service/errors" @@ -64,10 +65,11 @@ type QuerierOptions struct { FeatureLookup interfaces.FeatureLookup // used for testing - TestingMode bool - ReturnedSeries []*v3.Series - ReturnedErr error - UseLogsNewSchema bool + TestingMode bool + ReturnedSeries []*v3.Series + ReturnedErr error + UseLogsNewSchema bool + UseTraceNewSchema bool } func NewQuerier(opts QuerierOptions) interfaces.Querier { @@ -76,6 +78,11 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { logsQueryBuilder = logsV4.PrepareLogsQuery } + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if opts.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } + qc := querycache.NewQueryCache(querycache.WithCache(opts.Cache), querycache.WithFluxInterval(opts.FluxInterval)) return &querier{ @@ -86,16 +93,17 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { fluxInterval: opts.FluxInterval, builder: queryBuilder.NewQueryBuilder(queryBuilder.QueryBuilderOptions{ - BuildTraceQuery: tracesV3.PrepareTracesQuery, + BuildTraceQuery: tracesQueryBuilder, BuildLogQuery: logsQueryBuilder, BuildMetricQuery: metricsV4.PrepareMetricQuery, }, opts.FeatureLookup), featureLookUp: opts.FeatureLookup, - testingMode: opts.TestingMode, - returnedSeries: opts.ReturnedSeries, - returnedErr: opts.ReturnedErr, - UseLogsNewSchema: opts.UseLogsNewSchema, + testingMode: opts.TestingMode, + returnedSeries: opts.ReturnedSeries, + returnedErr: opts.ReturnedErr, + UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, } } diff --git a/pkg/query-service/app/querier/v2/querier_test.go b/pkg/query-service/app/querier/v2/querier_test.go index e52bb53823..75defa88ef 100644 --- a/pkg/query-service/app/querier/v2/querier_test.go +++ b/pkg/query-service/app/querier/v2/querier_test.go @@ -1437,6 +1437,7 @@ func Test_querier_runWindowBasedListQuery(t *testing.T) { featureManager.StartManager(), "", true, + true, ) q := &querier{ diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 6a030e73cf..76637ad328 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -67,6 +67,7 @@ type ServerOptions struct { FluxInterval string Cluster string UseLogsNewSchema bool + UseTraceNewSchema bool } // Server runs HTTP, Mux and a grpc server @@ -130,6 +131,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.DialTimeout, serverOptions.Cluster, serverOptions.UseLogsNewSchema, + serverOptions.UseTraceNewSchema, ) go clickhouseReader.Start(readerReady) reader = clickhouseReader @@ -157,7 +159,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { rm, err := makeRulesManager( serverOptions.PromConfigPath, constants.GetAlertManagerApiPrefix(), - serverOptions.RuleRepoURL, localDB, reader, c, serverOptions.DisableRules, fm, serverOptions.UseLogsNewSchema) + serverOptions.RuleRepoURL, localDB, reader, c, serverOptions.DisableRules, fm, serverOptions.UseLogsNewSchema, serverOptions.UseTraceNewSchema) if err != nil { return nil, err } @@ -202,6 +204,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { Cache: c, FluxInterval: fluxInterval, UseLogsNewSchema: serverOptions.UseLogsNewSchema, + UseTraceNewSchema: serverOptions.UseTraceNewSchema, }) if err != nil { return nil, err @@ -727,7 +730,8 @@ func makeRulesManager( cache cache.Cache, disableRules bool, fm interfaces.FeatureLookup, - useLogsNewSchema bool) (*rules.Manager, error) { + useLogsNewSchema bool, + useTraceNewSchema bool) (*rules.Manager, error) { // create engine pqle, err := pqle.FromReader(ch) @@ -744,18 +748,19 @@ func makeRulesManager( // create manager opts managerOpts := &rules.ManagerOptions{ - NotifierOpts: notifierOpts, - PqlEngine: pqle, - RepoURL: ruleRepoURL, - DBConn: db, - Context: context.Background(), - Logger: zap.L(), - DisableRules: disableRules, - FeatureFlags: fm, - Reader: ch, - Cache: cache, - EvalDelay: constants.GetEvalDelay(), - UseLogsNewSchema: useLogsNewSchema, + NotifierOpts: notifierOpts, + PqlEngine: pqle, + RepoURL: ruleRepoURL, + DBConn: db, + Context: context.Background(), + Logger: zap.L(), + DisableRules: disableRules, + FeatureFlags: fm, + Reader: ch, + Cache: cache, + EvalDelay: constants.GetEvalDelay(), + UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } // create Manager diff --git a/pkg/query-service/main.go b/pkg/query-service/main.go index cd6a542220..1f1c54b961 100644 --- a/pkg/query-service/main.go +++ b/pkg/query-service/main.go @@ -39,6 +39,7 @@ func main() { var disableRules bool var useLogsNewSchema bool + var useTraceNewSchema bool // the url used to build link in the alert messages in slack and other systems var ruleRepoURL, cacheConfigPath, fluxInterval string var cluster string @@ -50,6 +51,7 @@ func main() { var dialTimeout time.Duration flag.BoolVar(&useLogsNewSchema, "use-logs-new-schema", false, "use logs_v2 schema for logs") + flag.BoolVar(&useTraceNewSchema, "use-trace-new-schema", false, "use new schema for traces") flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") flag.StringVar(&skipTopLvlOpsPath, "skip-top-level-ops", "", "(config file to skip top level operations)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") @@ -87,6 +89,7 @@ func main() { FluxInterval: fluxInterval, Cluster: cluster, UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } // Read the jwt secret key diff --git a/pkg/query-service/rules/manager.go b/pkg/query-service/rules/manager.go index 50ad7b5430..84e055cfa0 100644 --- a/pkg/query-service/rules/manager.go +++ b/pkg/query-service/rules/manager.go @@ -35,7 +35,8 @@ type PrepareTaskOptions struct { ManagerOpts *ManagerOptions NotifyFunc NotifyFunc - UseLogsNewSchema bool + UseLogsNewSchema bool + UseTraceNewSchema bool } type PrepareTestRuleOptions struct { @@ -48,7 +49,8 @@ type PrepareTestRuleOptions struct { ManagerOpts *ManagerOptions NotifyFunc NotifyFunc - UseLogsNewSchema bool + UseLogsNewSchema bool + UseTraceNewSchema bool } const taskNamesuffix = "webAppEditor" @@ -91,9 +93,9 @@ type ManagerOptions struct { PrepareTaskFunc func(opts PrepareTaskOptions) (Task, error) + UseLogsNewSchema bool + UseTraceNewSchema bool PrepareTestRuleFunc func(opts PrepareTestRuleOptions) (int, *model.ApiError) - - UseLogsNewSchema bool } // The Manager manages recording and alerting rules. @@ -117,7 +119,8 @@ type Manager struct { prepareTaskFunc func(opts PrepareTaskOptions) (Task, error) prepareTestRuleFunc func(opts PrepareTestRuleOptions) (int, *model.ApiError) - UseLogsNewSchema bool + UseLogsNewSchema bool + UseTraceNewSchema bool } func defaultOptions(o *ManagerOptions) *ManagerOptions { @@ -156,6 +159,7 @@ func defaultPrepareTaskFunc(opts PrepareTaskOptions) (Task, error) { opts.FF, opts.Reader, opts.UseLogsNewSchema, + opts.UseTraceNewSchema, WithEvalDelay(opts.ManagerOpts.EvalDelay), ) @@ -368,7 +372,8 @@ func (m *Manager) editTask(rule *PostableRule, taskName string) error { ManagerOpts: m.opts, NotifyFunc: m.prepareNotifyFunc(), - UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseTraceNewSchema: m.opts.UseTraceNewSchema, }) if err != nil { @@ -490,7 +495,8 @@ func (m *Manager) addTask(rule *PostableRule, taskName string) error { ManagerOpts: m.opts, NotifyFunc: m.prepareNotifyFunc(), - UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseTraceNewSchema: m.opts.UseTraceNewSchema, }) if err != nil { @@ -809,15 +815,16 @@ func (m *Manager) TestNotification(ctx context.Context, ruleStr string) (int, *m } alertCount, apiErr := m.prepareTestRuleFunc(PrepareTestRuleOptions{ - Rule: parsedRule, - RuleDB: m.ruleDB, - Logger: m.logger, - Reader: m.reader, - Cache: m.cache, - FF: m.featureFlags, - ManagerOpts: m.opts, - NotifyFunc: m.prepareNotifyFunc(), - UseLogsNewSchema: m.opts.UseLogsNewSchema, + Rule: parsedRule, + RuleDB: m.ruleDB, + Logger: m.logger, + Reader: m.reader, + Cache: m.cache, + FF: m.featureFlags, + ManagerOpts: m.opts, + NotifyFunc: m.prepareNotifyFunc(), + UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseTraceNewSchema: m.opts.UseTraceNewSchema, }) return alertCount, apiErr diff --git a/pkg/query-service/rules/test_notification.go b/pkg/query-service/rules/test_notification.go index 37fb2e5f1b..e30b7db94f 100644 --- a/pkg/query-service/rules/test_notification.go +++ b/pkg/query-service/rules/test_notification.go @@ -49,6 +49,7 @@ func defaultTestNotification(opts PrepareTestRuleOptions) (int, *model.ApiError) opts.FF, opts.Reader, opts.UseLogsNewSchema, + opts.UseTraceNewSchema, WithSendAlways(), WithSendUnmatched(), ) diff --git a/pkg/query-service/rules/threshold_rule.go b/pkg/query-service/rules/threshold_rule.go index 3971597ec2..ad55083f8f 100644 --- a/pkg/query-service/rules/threshold_rule.go +++ b/pkg/query-service/rules/threshold_rule.go @@ -29,6 +29,7 @@ import ( logsv3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/formatter" yaml "gopkg.in/yaml.v2" @@ -50,6 +51,8 @@ type ThresholdRule struct { // used for attribute metadata enrichment for logs and traces logsKeys map[string]v3.AttributeKey spansKeys map[string]v3.AttributeKey + + useTraceNewSchema bool } func NewThresholdRule( @@ -58,6 +61,7 @@ func NewThresholdRule( featureFlags interfaces.FeatureLookup, reader interfaces.Reader, useLogsNewSchema bool, + useTraceNewSchema bool, opts ...RuleOption, ) (*ThresholdRule, error) { @@ -74,19 +78,21 @@ func NewThresholdRule( } querierOption := querier.QuerierOptions{ - Reader: reader, - Cache: nil, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FeatureLookup: featureFlags, - UseLogsNewSchema: useLogsNewSchema, + Reader: reader, + Cache: nil, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FeatureLookup: featureFlags, + UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } querierOptsV2 := querierV2.QuerierOptions{ - Reader: reader, - Cache: nil, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FeatureLookup: featureFlags, - UseLogsNewSchema: useLogsNewSchema, + Reader: reader, + Cache: nil, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FeatureLookup: featureFlags, + UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } t.querier = querier.NewQuerier(querierOption) @@ -296,7 +302,11 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, ts time.Time) (Vec return nil, err } r.spansKeys = spanKeys - tracesV3.Enrich(params, spanKeys) + if r.useTraceNewSchema { + tracesV4.Enrich(params, spanKeys) + } else { + tracesV3.Enrich(params, spanKeys) + } } } diff --git a/pkg/query-service/rules/threshold_rule_test.go b/pkg/query-service/rules/threshold_rule_test.go index e75c82b1a0..5a4ab9c970 100644 --- a/pkg/query-service/rules/threshold_rule_test.go +++ b/pkg/query-service/rules/threshold_rule_test.go @@ -791,7 +791,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) { postableRule.RuleCondition.MatchType = MatchType(c.matchType) postableRule.RuleCondition.Target = &c.target - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -880,7 +880,7 @@ func TestPrepareLinksToLogs(t *testing.T) { } fm := featureManager.StartManager() - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -922,7 +922,7 @@ func TestPrepareLinksToTraces(t *testing.T) { } fm := featureManager.StartManager() - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -998,7 +998,7 @@ func TestThresholdRuleLabelNormalization(t *testing.T) { postableRule.RuleCondition.MatchType = MatchType(c.matchType) postableRule.RuleCondition.Target = &c.target - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -1051,7 +1051,7 @@ func TestThresholdRuleEvalDelay(t *testing.T) { fm := featureManager.StartManager() for idx, c := range cases { - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true) // no eval delay + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true) // no eval delay if err != nil { assert.NoError(t, err) } @@ -1100,7 +1100,7 @@ func TestThresholdRuleClickHouseTmpl(t *testing.T) { fm := featureManager.StartManager() for idx, c := range cases { - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -1118,7 +1118,7 @@ func TestThresholdRuleClickHouseTmpl(t *testing.T) { type queryMatcherAny struct { } -func (m *queryMatcherAny) Match(string, string) error { +func (m *queryMatcherAny) Match(x string, y string) error { return nil } @@ -1241,9 +1241,9 @@ func TestThresholdRuleUnitCombinations(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true, true) - rule, err := NewThresholdRule("69", &postableRule, fm, reader, true) + rule, err := NewThresholdRule("69", &postableRule, fm, reader, true, true) rule.TemporalityMap = map[string]map[v3.Temporality]bool{ "signoz_calls_total": { v3.Delta: true, @@ -1340,9 +1340,9 @@ func TestThresholdRuleNoData(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true, true) - rule, err := NewThresholdRule("69", &postableRule, fm, reader, true) + rule, err := NewThresholdRule("69", &postableRule, fm, reader, true, true) rule.TemporalityMap = map[string]map[v3.Temporality]bool{ "signoz_calls_total": { v3.Delta: true, @@ -1423,9 +1423,12 @@ func TestThresholdRuleTracesLink(t *testing.T) { for idx, c := range testCases { metaRows := cmock.NewRows(metaCols, c.metaValues) mock. - ExpectQuery("SELECT DISTINCT(tagKey), tagType, dataType, isColumn FROM archiveNamespace.span_attributes_keys"). + ExpectQuery("SELECT DISTINCT(tagKey), tagType, dataType FROM archiveNamespace.span_attributes_keys"). WillReturnRows(metaRows) + mock. + ExpectSelect("SHOW CREATE TABLE signoz_traces.distributed_signoz_index_v3").WillReturnRows(&cmock.Rows{}) + rows := cmock.NewRows(cols, c.values) // We are testing the eval logic after the query is run @@ -1445,9 +1448,9 @@ func TestThresholdRuleTracesLink(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true, true) - rule, err := NewThresholdRule("69", &postableRule, fm, reader, true) + rule, err := NewThresholdRule("69", &postableRule, fm, reader, true, true) rule.TemporalityMap = map[string]map[v3.Temporality]bool{ "signoz_calls_total": { v3.Delta: true, @@ -1570,9 +1573,9 @@ func TestThresholdRuleLogsLink(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true, true) - rule, err := NewThresholdRule("69", &postableRule, fm, reader, true) + rule, err := NewThresholdRule("69", &postableRule, fm, reader, true, true) rule.TemporalityMap = map[string]map[v3.Temporality]bool{ "signoz_calls_total": { v3.Delta: true, @@ -1648,7 +1651,7 @@ func TestThresholdRuleShiftBy(t *testing.T) { }, } - rule, err := NewThresholdRule("69", &postableRule, nil, nil, true) + rule, err := NewThresholdRule("69", &postableRule, nil, nil, true, true) if err != nil { assert.NoError(t, err) } diff --git a/pkg/query-service/tests/integration/test_utils.go b/pkg/query-service/tests/integration/test_utils.go index d060433dba..c1c71808a9 100644 --- a/pkg/query-service/tests/integration/test_utils.go +++ b/pkg/query-service/tests/integration/test_utils.go @@ -46,6 +46,7 @@ func NewMockClickhouseReader( featureFlags, "", true, + true, ) return reader, mockDB