From 2c30e1493fc6f036208cf4a1e55b5311bab4f473 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 21 Oct 2024 18:20:48 +0530 Subject: [PATCH 01/41] feat: trace v4 inital commit --- ee/query-service/app/api/api.go | 6 +- ee/query-service/app/db/reader.go | 3 +- ee/query-service/app/server.go | 12 +- ee/query-service/main.go | 3 + ee/query-service/rules/manager.go | 1 + .../app/clickhouseReader/reader.go | 8 +- pkg/query-service/app/http_handler.go | 32 +- .../app/logs/v4/query_builder.go | 5 +- pkg/query-service/app/querier/querier.go | 21 +- pkg/query-service/app/querier/v2/querier.go | 18 +- .../v4 => resource}/resource_query_builder.go | 8 +- .../resource_query_builder_test.go | 8 +- pkg/query-service/app/server.go | 33 +- .../app/traces/v3/query_builder.go | 48 +- pkg/query-service/app/traces/v4/enrich.go | 52 ++ .../app/traces/v4/enrich_test.go | 57 ++ .../app/traces/v4/query_builder.go | 435 ++++++++++++++ .../app/traces/v4/query_builder_test.go | 540 ++++++++++++++++++ pkg/query-service/main.go | 3 + 19 files changed, 1209 insertions(+), 84 deletions(-) rename pkg/query-service/app/{logs/v4 => resource}/resource_query_builder.go (95%) rename pkg/query-service/app/{logs/v4 => resource}/resource_query_builder_test.go (97%) create mode 100644 pkg/query-service/app/traces/v4/enrich.go create mode 100644 pkg/query-service/app/traces/v4/enrich_test.go create mode 100644 pkg/query-service/app/traces/v4/query_builder.go create mode 100644 pkg/query-service/app/traces/v4/query_builder_test.go diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go index 82557705fd..5b7ca7cbb7 100644 --- a/ee/query-service/app/api/api.go +++ b/ee/query-service/app/api/api.go @@ -38,8 +38,9 @@ type APIHandlerOptions struct { Cache cache.Cache Gateway *httputil.ReverseProxy // Querier Influx Interval - FluxInterval time.Duration - UseLogsNewSchema bool + FluxInterval time.Duration + UseLogsNewSchema bool + UseTraceNewSchema bool } type APIHandler struct { @@ -65,6 +66,7 @@ func NewAPIHandler(opts APIHandlerOptions) (*APIHandler, error) { Cache: opts.Cache, FluxInterval: opts.FluxInterval, UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, }) if err != nil { diff --git a/ee/query-service/app/db/reader.go b/ee/query-service/app/db/reader.go index fcab1cb991..9794abd013 100644 --- a/ee/query-service/app/db/reader.go +++ b/ee/query-service/app/db/reader.go @@ -26,8 +26,9 @@ func NewDataConnector( dialTimeout time.Duration, cluster string, useLogsNewSchema bool, + useTraceNewSchema bool, ) *ClickhouseReader { - ch := basechr.NewReader(localDB, promConfigPath, lm, maxIdleConns, maxOpenConns, dialTimeout, cluster, useLogsNewSchema) + ch := basechr.NewReader(localDB, promConfigPath, lm, maxIdleConns, maxOpenConns, dialTimeout, cluster, useLogsNewSchema, useTraceNewSchema) return &ClickhouseReader{ conn: ch.GetConn(), appdb: localDB, diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index cf54693ba8..e63a3c7893 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -78,6 +78,7 @@ type ServerOptions struct { Cluster string GatewayUrl string UseLogsNewSchema bool + UseTraceNewSchema bool } // Server runs HTTP api service @@ -156,6 +157,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.DialTimeout, serverOptions.Cluster, serverOptions.UseLogsNewSchema, + serverOptions.UseTraceNewSchema, ) go qb.Start(readerReady) reader = qb @@ -189,6 +191,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.DisableRules, lm, serverOptions.UseLogsNewSchema, + serverOptions.UseTraceNewSchema, ) if err != nil { @@ -270,6 +273,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { FluxInterval: fluxInterval, Gateway: gatewayProxy, UseLogsNewSchema: serverOptions.UseLogsNewSchema, + UseTraceNewSchema: serverOptions.UseTraceNewSchema, } apiHandler, err := api.NewAPIHandler(apiOpts) @@ -736,7 +740,8 @@ func makeRulesManager( cache cache.Cache, disableRules bool, fm baseint.FeatureLookup, - useLogsNewSchema bool) (*baserules.Manager, error) { + useLogsNewSchema bool, + useTraceNewSchema bool) (*baserules.Manager, error) { // create engine pqle, err := pqle.FromConfigPath(promConfigPath) @@ -765,8 +770,9 @@ func makeRulesManager( Cache: cache, EvalDelay: baseconst.GetEvalDelay(), - PrepareTaskFunc: rules.PrepareTaskFunc, - UseLogsNewSchema: useLogsNewSchema, + PrepareTaskFunc: rules.PrepareTaskFunc, + UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } // create Manager diff --git a/ee/query-service/main.go b/ee/query-service/main.go index 41cc69aa49..dd52ab73a5 100644 --- a/ee/query-service/main.go +++ b/ee/query-service/main.go @@ -94,6 +94,7 @@ func main() { var cluster string var useLogsNewSchema bool + var useTraceNewSchema bool var cacheConfigPath, fluxInterval string var enableQueryServiceLogOTLPExport bool var preferSpanMetrics bool @@ -104,6 +105,7 @@ func main() { var gatewayUrl string flag.BoolVar(&useLogsNewSchema, "use-logs-new-schema", false, "use logs_v2 schema for logs") + flag.BoolVar(&useTraceNewSchema, "use-trace-new-schema", false, "use new schema for traces") flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") flag.StringVar(&skipTopLvlOpsPath, "skip-top-level-ops", "", "(config file to skip top level operations)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") @@ -143,6 +145,7 @@ func main() { Cluster: cluster, GatewayUrl: gatewayUrl, UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } // Read the jwt secret key diff --git a/ee/query-service/rules/manager.go b/ee/query-service/rules/manager.go index e44bbcf82b..d0365740c2 100644 --- a/ee/query-service/rules/manager.go +++ b/ee/query-service/rules/manager.go @@ -21,6 +21,7 @@ func PrepareTaskFunc(opts baserules.PrepareTaskOptions) (baserules.Task, error) opts.FF, opts.Reader, opts.UseLogsNewSchema, + opts.UseTraceNewSchema, baserules.WithEvalDelay(opts.ManagerOpts.EvalDelay), ) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 6038ed602f..cc2fc2d7f6 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -146,6 +146,7 @@ type ClickHouseReader struct { cluster string useLogsNewSchema bool + useTraceNewSchema bool logsTableName string logsLocalTableName string } @@ -160,6 +161,7 @@ func NewReader( dialTimeout time.Duration, cluster string, useLogsNewSchema bool, + useTraceNewSchema bool, ) *ClickHouseReader { datasource := os.Getenv("ClickHouseUrl") @@ -170,7 +172,7 @@ func NewReader( zap.L().Fatal("failed to initialize ClickHouse", zap.Error(err)) } - return NewReaderFromClickhouseConnection(db, options, localDB, configFile, featureFlag, cluster, useLogsNewSchema) + return NewReaderFromClickhouseConnection(db, options, localDB, configFile, featureFlag, cluster, useLogsNewSchema, useTraceNewSchema) } func NewReaderFromClickhouseConnection( @@ -181,6 +183,7 @@ func NewReaderFromClickhouseConnection( featureFlag interfaces.FeatureLookup, cluster string, useLogsNewSchema bool, + useTraceNewSchema bool, ) *ClickHouseReader { alertManager, err := am.New() if err != nil { @@ -245,7 +248,8 @@ func NewReaderFromClickhouseConnection( cluster: cluster, queryProgressTracker: queryprogress.NewQueryProgressTracker(), - useLogsNewSchema: useLogsNewSchema, + useLogsNewSchema: useLogsNewSchema, + useTraceNewSchema: useTraceNewSchema, logsTableV2: options.primary.LogsTableV2, logsLocalTableV2: options.primary.LogsLocalTableV2, diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 790e64fb60..f34e8055f8 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -110,7 +110,8 @@ type APIHandler struct { // Websocket connection upgrader Upgrader *websocket.Upgrader - UseLogsNewSchema bool + UseLogsNewSchema bool + UseTraceNewSchema bool hostsRepo *inframetrics.HostsRepo processesRepo *inframetrics.ProcessesRepo @@ -152,6 +153,8 @@ type APIHandlerOpts struct { // Use Logs New schema UseLogsNewSchema bool + + UseTraceNewSchema bool } // NewAPIHandler returns an APIHandler @@ -163,21 +166,23 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { } querierOpts := querier.QuerierOptions{ - Reader: opts.Reader, - Cache: opts.Cache, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FluxInterval: opts.FluxInterval, - FeatureLookup: opts.FeatureFlags, - UseLogsNewSchema: opts.UseLogsNewSchema, + Reader: opts.Reader, + Cache: opts.Cache, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FluxInterval: opts.FluxInterval, + FeatureLookup: opts.FeatureFlags, + UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, } querierOptsV2 := querierV2.QuerierOptions{ - Reader: opts.Reader, - Cache: opts.Cache, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FluxInterval: opts.FluxInterval, - FeatureLookup: opts.FeatureFlags, - UseLogsNewSchema: opts.UseLogsNewSchema, + Reader: opts.Reader, + Cache: opts.Cache, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FluxInterval: opts.FluxInterval, + FeatureLookup: opts.FeatureFlags, + UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, } querier := querier.NewQuerier(querierOpts) @@ -203,6 +208,7 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { querier: querier, querierV2: querierv2, UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, hostsRepo: hostsRepo, processesRepo: processesRepo, } diff --git a/pkg/query-service/app/logs/v4/query_builder.go b/pkg/query-service/app/logs/v4/query_builder.go index 49e585e64b..7420c3c091 100644 --- a/pkg/query-service/app/logs/v4/query_builder.go +++ b/pkg/query-service/app/logs/v4/query_builder.go @@ -5,6 +5,7 @@ import ( "strings" logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + "go.signoz.io/signoz/pkg/query-service/app/resource" "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/utils" @@ -372,7 +373,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build } // build the where clause for resource table - resourceSubQuery, err := buildResourceSubQuery(bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) + resourceSubQuery, err := resource.BuildResourceSubQuery("signoz_logs", DISTRIBUTED_LOGS_V2_RESOURCE, bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) if err != nil { return "", err } @@ -463,7 +464,7 @@ func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) { } // no values for bucket start and end - resourceSubQuery, err := buildResourceSubQuery(0, 0, mq.Filters, mq.GroupBy, mq.AggregateAttribute, true) + resourceSubQuery, err := resource.BuildResourceSubQuery("signoz_logs", DISTRIBUTED_LOGS_V2_RESOURCE, 0, 0, mq.Filters, mq.GroupBy, mq.AggregateAttribute, true) if err != nil { return "", err } diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index fd7198b334..02234bd037 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -52,7 +52,8 @@ type querier struct { returnedSeries []*v3.Series returnedErr error - UseLogsNewSchema bool + UseLogsNewSchema bool + UseTraceNewSchema bool } type QuerierOptions struct { @@ -63,10 +64,11 @@ type QuerierOptions struct { FeatureLookup interfaces.FeatureLookup // used for testing - TestingMode bool - ReturnedSeries []*v3.Series - ReturnedErr error - UseLogsNewSchema bool + TestingMode bool + ReturnedSeries []*v3.Series + ReturnedErr error + UseLogsNewSchema bool + UseTraceNewSchema bool } func NewQuerier(opts QuerierOptions) interfaces.Querier { @@ -91,10 +93,11 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { }, opts.FeatureLookup), featureLookUp: opts.FeatureLookup, - testingMode: opts.TestingMode, - returnedSeries: opts.ReturnedSeries, - returnedErr: opts.ReturnedErr, - UseLogsNewSchema: opts.UseLogsNewSchema, + testingMode: opts.TestingMode, + returnedSeries: opts.ReturnedSeries, + returnedErr: opts.ReturnedErr, + UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, } } diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 311d213656..a524591c6b 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -48,10 +48,11 @@ type querier struct { testingMode bool queriesExecuted []string // tuple of start and end time in milliseconds - timeRanges [][]int - returnedSeries []*v3.Series - returnedErr error - UseLogsNewSchema bool + timeRanges [][]int + returnedSeries []*v3.Series + returnedErr error + UseLogsNewSchema bool + UseTraceNewSchema bool } type QuerierOptions struct { @@ -62,10 +63,11 @@ type QuerierOptions struct { FeatureLookup interfaces.FeatureLookup // used for testing - TestingMode bool - ReturnedSeries []*v3.Series - ReturnedErr error - UseLogsNewSchema bool + TestingMode bool + ReturnedSeries []*v3.Series + ReturnedErr error + UseLogsNewSchema bool + UseTraceNewSchema bool } func NewQuerier(opts QuerierOptions) interfaces.Querier { diff --git a/pkg/query-service/app/logs/v4/resource_query_builder.go b/pkg/query-service/app/resource/resource_query_builder.go similarity index 95% rename from pkg/query-service/app/logs/v4/resource_query_builder.go rename to pkg/query-service/app/resource/resource_query_builder.go index 3f7f2682cb..2b128de3d4 100644 --- a/pkg/query-service/app/logs/v4/resource_query_builder.go +++ b/pkg/query-service/app/resource/resource_query_builder.go @@ -1,4 +1,4 @@ -package v4 +package resource import ( "fmt" @@ -211,7 +211,7 @@ func buildResourceFiltersFromAggregateAttribute(aggregateAttribute v3.AttributeK return "" } -func buildResourceSubQuery(bucketStart, bucketEnd int64, fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey, isLiveTail bool) (string, error) { +func BuildResourceSubQuery(database, table string, bucketStart, bucketEnd int64, fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey, isLiveTail bool) (string, error) { // BUILD THE WHERE CLAUSE var conditions []string @@ -242,10 +242,10 @@ func buildResourceSubQuery(bucketStart, bucketEnd int64, fs *v3.FilterSet, group // BUILD THE FINAL QUERY var query string if isLiveTail { - query = fmt.Sprintf("SELECT fingerprint FROM signoz_logs.%s WHERE ", DISTRIBUTED_LOGS_V2_RESOURCE) + query = fmt.Sprintf("SELECT fingerprint FROM %s.%s WHERE ", database, table) query = "(" + query + conditionStr } else { - query = fmt.Sprintf("SELECT fingerprint FROM signoz_logs.%s WHERE (seen_at_ts_bucket_start >= %d) AND (seen_at_ts_bucket_start <= %d) AND ", DISTRIBUTED_LOGS_V2_RESOURCE, bucketStart, bucketEnd) + query = fmt.Sprintf("SELECT fingerprint FROM %s.%s WHERE (seen_at_ts_bucket_start >= %d) AND (seen_at_ts_bucket_start <= %d) AND ", database, table, bucketStart, bucketEnd) query = "(" + query + conditionStr + ")" } diff --git a/pkg/query-service/app/logs/v4/resource_query_builder_test.go b/pkg/query-service/app/resource/resource_query_builder_test.go similarity index 97% rename from pkg/query-service/app/logs/v4/resource_query_builder_test.go rename to pkg/query-service/app/resource/resource_query_builder_test.go index 7bea0bc291..d48b0bb840 100644 --- a/pkg/query-service/app/logs/v4/resource_query_builder_test.go +++ b/pkg/query-service/app/resource/resource_query_builder_test.go @@ -1,4 +1,4 @@ -package v4 +package resource import ( "reflect" @@ -434,6 +434,8 @@ func Test_buildResourceFiltersFromAggregateAttribute(t *testing.T) { func Test_buildResourceSubQuery(t *testing.T) { type args struct { + database string + table string bucketStart int64 bucketEnd int64 fs *v3.FilterSet @@ -449,6 +451,8 @@ func Test_buildResourceSubQuery(t *testing.T) { { name: "build sub query", args: args{ + database: "signoz_logs", + table: "distributed_logs_v2_resource", bucketStart: 1680064560, bucketEnd: 1680066458, fs: &v3.FilterSet{ @@ -497,7 +501,7 @@ func Test_buildResourceSubQuery(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := buildResourceSubQuery(tt.args.bucketStart, tt.args.bucketEnd, tt.args.fs, tt.args.groupBy, tt.args.aggregateAttribute, false) + got, err := BuildResourceSubQuery(tt.args.database, tt.args.table, tt.args.bucketStart, tt.args.bucketEnd, tt.args.fs, tt.args.groupBy, tt.args.aggregateAttribute, false) if (err != nil) != tt.wantErr { t.Errorf("buildResourceSubQuery() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index dc6ac21e15..8eef01dbac 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -67,6 +67,7 @@ type ServerOptions struct { FluxInterval string Cluster string UseLogsNewSchema bool + UseTraceNewSchema bool } // Server runs HTTP, Mux and a grpc server @@ -130,6 +131,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { serverOptions.DialTimeout, serverOptions.Cluster, serverOptions.UseLogsNewSchema, + serverOptions.UseTraceNewSchema, ) go clickhouseReader.Start(readerReady) reader = clickhouseReader @@ -157,7 +159,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { rm, err := makeRulesManager( serverOptions.PromConfigPath, constants.GetAlertManagerApiPrefix(), - serverOptions.RuleRepoURL, localDB, reader, c, serverOptions.DisableRules, fm, serverOptions.UseLogsNewSchema) + serverOptions.RuleRepoURL, localDB, reader, c, serverOptions.DisableRules, fm, serverOptions.UseLogsNewSchema, serverOptions.UseTraceNewSchema) if err != nil { return nil, err } @@ -202,6 +204,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { Cache: c, FluxInterval: fluxInterval, UseLogsNewSchema: serverOptions.UseLogsNewSchema, + UseTraceNewSchema: serverOptions.UseTraceNewSchema, }) if err != nil { return nil, err @@ -721,7 +724,8 @@ func makeRulesManager( cache cache.Cache, disableRules bool, fm interfaces.FeatureLookup, - useLogsNewSchema bool) (*rules.Manager, error) { + useLogsNewSchema bool, + useTraceNewSchema bool) (*rules.Manager, error) { // create engine pqle, err := pqle.FromReader(ch) @@ -738,18 +742,19 @@ func makeRulesManager( // create manager opts managerOpts := &rules.ManagerOptions{ - NotifierOpts: notifierOpts, - PqlEngine: pqle, - RepoURL: ruleRepoURL, - DBConn: db, - Context: context.Background(), - Logger: zap.L(), - DisableRules: disableRules, - FeatureFlags: fm, - Reader: ch, - Cache: cache, - EvalDelay: constants.GetEvalDelay(), - UseLogsNewSchema: useLogsNewSchema, + NotifierOpts: notifierOpts, + PqlEngine: pqle, + RepoURL: ruleRepoURL, + DBConn: db, + Context: context.Background(), + Logger: zap.L(), + DisableRules: disableRules, + FeatureFlags: fm, + Reader: ch, + Cache: cache, + EvalDelay: constants.GetEvalDelay(), + UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } // create Manager diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index fcb682cfbb..efc48b40b6 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -15,7 +15,7 @@ type Options struct { PreferRPM bool } -var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ +var AggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ v3.AggregateOperatorP05: 0.05, v3.AggregateOperatorP10: 0.10, v3.AggregateOperatorP20: 0.20, @@ -27,7 +27,7 @@ var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ v3.AggregateOperatorP99: 0.99, } -var aggregateOperatorToSQLFunc = map[v3.AggregateOperator]string{ +var AggregateOperatorToSQLFunc = map[v3.AggregateOperator]string{ v3.AggregateOperatorAvg: "avg", v3.AggregateOperatorMax: "max", v3.AggregateOperatorMin: "min", @@ -114,7 +114,7 @@ func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.Attri return selectLabels } -func getSelectKeys(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string { +func GetSelectKeys(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string { var selectLabels []string if aggregatorOperator == v3.AggregateOperatorNoOp { return "" @@ -178,7 +178,7 @@ func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) { conditions = append(conditions, fmt.Sprintf(operator, columnName, fmtVal)) case v3.FilterOperatorExists, v3.FilterOperatorNotExists: if item.Key.IsColumn { - subQuery, err := existsSubQueryForFixedColumn(item.Key, item.Operator) + subQuery, err := ExistsSubQueryForFixedColumn(item.Key, item.Operator) if err != nil { return "", err } @@ -204,7 +204,7 @@ func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) { return queryString, nil } -func existsSubQueryForFixedColumn(key v3.AttributeKey, op v3.FilterOperator) (string, error) { +func ExistsSubQueryForFixedColumn(key v3.AttributeKey, op v3.FilterOperator) (string, error) { if key.DataType == v3.AttributeKeyDataTypeString { if op == v3.FilterOperatorExists { return fmt.Sprintf("%s %s ''", key.Key, tracesOperatorMappingV3[v3.FilterOperatorNotEqual]), nil @@ -249,7 +249,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, pan selectLabels := getSelectLabels(mq.AggregateOperator, mq.GroupBy) - having := having(mq.Having) + having := Having(mq.Having) if having != "" { having = " having " + having } @@ -277,7 +277,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, pan // we don't need value for first query if options.GraphLimitQtype == constants.FirstQueryGraphLimit { - queryTmpl = "SELECT " + getSelectKeys(mq.AggregateOperator, mq.GroupBy) + " from (" + queryTmpl + ")" + queryTmpl = "SELECT " + GetSelectKeys(mq.AggregateOperator, mq.GroupBy) + " from (" + queryTmpl + ")" } emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(mq.GroupBy) @@ -286,7 +286,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, pan } filterSubQuery += emptyValuesInGroupByFilter - groupBy := groupByAttributeKeyTags(panelType, options.GraphLimitQtype, mq.GroupBy...) + groupBy := GroupByAttributeKeyTags(panelType, options.GraphLimitQtype, mq.GroupBy...) if groupBy != "" { groupBy = " group by " + groupBy } @@ -296,7 +296,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, pan } if options.GraphLimitQtype == constants.SecondQueryGraphLimit { - filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", getSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "%s)" + filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", GetSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "%s)" } aggregationKey := "" @@ -316,7 +316,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, pan rate = rate / 60.0 } - op := fmt.Sprintf("%s(%s)/%f", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, rate) + op := fmt.Sprintf("%s(%s)/%f", AggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, rate) query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) return query, nil case @@ -329,17 +329,17 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, pan v3.AggregateOperatorP90, v3.AggregateOperatorP95, v3.AggregateOperatorP99: - op := fmt.Sprintf("quantile(%v)(%s)", aggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey) + op := fmt.Sprintf("quantile(%v)(%s)", AggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey) query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorAvg, v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax: - op := fmt.Sprintf("%s(%s)", aggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey) + op := fmt.Sprintf("%s(%s)", AggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey) query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) return query, nil case v3.AggregateOperatorCount: if mq.AggregateAttribute.Key != "" { if mq.AggregateAttribute.IsColumn { - subQuery, err := existsSubQueryForFixedColumn(mq.AggregateAttribute, v3.FilterOperatorExists) + subQuery, err := ExistsSubQueryForFixedColumn(mq.AggregateAttribute, v3.FilterOperatorExists) if err == nil { filterSubQuery = fmt.Sprintf("%s AND %s", filterSubQuery, subQuery) } @@ -359,9 +359,9 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, pan var query string if panelType == v3.PanelTypeTrace { withSubQuery := fmt.Sprintf(constants.TracesExplorerViewSQLSelectWithSubQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_LOCAL_TABLENAME, spanIndexTableTimeFilter, filterSubQuery) - withSubQuery = addLimitToQuery(withSubQuery, mq.Limit) + withSubQuery = AddLimitToQuery(withSubQuery, mq.Limit) if mq.Offset != 0 { - withSubQuery = addOffsetToQuery(withSubQuery, mq.Offset) + withSubQuery = AddOffsetToQuery(withSubQuery, mq.Offset) } // query = withSubQuery + ") " + fmt.Sprintf(constants.TracesExplorerViewSQLSelectQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_TABLENAME, constants.SIGNOZ_SPAN_INDEX_TABLENAME) query = fmt.Sprintf(constants.TracesExplorerViewSQLSelectBeforeSubQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_TABLENAME) + withSubQuery + ") " + fmt.Sprintf(constants.TracesExplorerViewSQLSelectAfterSubQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_TABLENAME, spanIndexTableTimeFilter) @@ -408,7 +408,7 @@ func groupBy(panelType v3.PanelType, graphLimitQtype string, tags ...string) str return strings.Join(tags, ",") } -func groupByAttributeKeyTags(panelType v3.PanelType, graphLimitQtype string, tags ...v3.AttributeKey) string { +func GroupByAttributeKeyTags(panelType v3.PanelType, graphLimitQtype string, tags ...v3.AttributeKey) string { groupTags := []string{} for _, tag := range tags { groupTags = append(groupTags, fmt.Sprintf("`%s`", tag.Key)) @@ -461,7 +461,7 @@ func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags [] return str } -func having(items []v3.Having) string { +func Having(items []v3.Having) string { // aggregate something and filter on that aggregate var having []string for _, item := range items { @@ -470,7 +470,7 @@ func having(items []v3.Having) string { return strings.Join(having, " AND ") } -func reduceToQuery(query string, reduceTo v3.ReduceToOperator, _ v3.AggregateOperator) (string, error) { +func ReduceToQuery(query string, reduceTo v3.ReduceToOperator, _ v3.AggregateOperator) (string, error) { var groupBy string switch reduceTo { @@ -490,14 +490,14 @@ func reduceToQuery(query string, reduceTo v3.ReduceToOperator, _ v3.AggregateOpe return query, nil } -func addLimitToQuery(query string, limit uint64) string { +func AddLimitToQuery(query string, limit uint64) string { if limit == 0 { limit = 100 } return fmt.Sprintf("%s LIMIT %d", query, limit) } -func addOffsetToQuery(query string, offset uint64) string { +func AddOffsetToQuery(query string, offset uint64) string { return fmt.Sprintf("%s OFFSET %d", query, offset) } @@ -514,7 +514,7 @@ func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.Builder if err != nil { return "", err } - query = addLimitToQuery(query, mq.Limit) + query = AddLimitToQuery(query, mq.Limit) return query, nil } else if options.GraphLimitQtype == constants.SecondQueryGraphLimit { @@ -530,13 +530,13 @@ func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.Builder return "", err } if panelType == v3.PanelTypeValue { - query, err = reduceToQuery(query, mq.ReduceTo, mq.AggregateOperator) + query, err = ReduceToQuery(query, mq.ReduceTo, mq.AggregateOperator) } if panelType == v3.PanelTypeList || panelType == v3.PanelTypeTable { - query = addLimitToQuery(query, mq.Limit) + query = AddLimitToQuery(query, mq.Limit) if mq.Offset != 0 { - query = addOffsetToQuery(query, mq.Offset) + query = AddOffsetToQuery(query, mq.Offset) } } return query, err diff --git a/pkg/query-service/app/traces/v4/enrich.go b/pkg/query-service/app/traces/v4/enrich.go new file mode 100644 index 0000000000..46935aef3a --- /dev/null +++ b/pkg/query-service/app/traces/v4/enrich.go @@ -0,0 +1,52 @@ +package v4 + +import v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + +func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) v3.AttributeKey { + // TODO(nitya) : update logic similar to logs + if key.Type == "" || key.DataType == "" { + // check if the key is present in the keys map + if existingKey, ok := keys[key.Key]; ok { + key.IsColumn = existingKey.IsColumn + key.Type = existingKey.Type + key.DataType = existingKey.DataType + } else { // if not present then set the default values + key.Type = v3.AttributeKeyTypeTag + key.DataType = v3.AttributeKeyDataTypeString + key.IsColumn = false + return key + } + } + return key +} + +func Enrich(params *v3.QueryRangeParamsV3, keys map[string]v3.AttributeKey) { + if params.CompositeQuery.QueryType == v3.QueryTypeBuilder { + for _, query := range params.CompositeQuery.BuilderQueries { + if query.DataSource == v3.DataSourceTraces { + EnrichTracesQuery(query, keys) + } + } + } +} + +func EnrichTracesQuery(query *v3.BuilderQuery, keys map[string]v3.AttributeKey) { + // enrich aggregate attribute + query.AggregateAttribute = enrichKeyWithMetadata(query.AggregateAttribute, keys) + // enrich filter items + if query.Filters != nil && len(query.Filters.Items) > 0 { + for idx, filter := range query.Filters.Items { + query.Filters.Items[idx].Key = enrichKeyWithMetadata(filter.Key, keys) + } + } + // enrich group by + for idx, groupBy := range query.GroupBy { + query.GroupBy[idx] = enrichKeyWithMetadata(groupBy, keys) + } + // enrich order by + query.OrderBy = enrichOrderBy(query.OrderBy, keys) + // enrich select columns + for idx, selectColumn := range query.SelectColumns { + query.SelectColumns[idx] = enrichKeyWithMetadata(selectColumn, keys) + } +} diff --git a/pkg/query-service/app/traces/v4/enrich_test.go b/pkg/query-service/app/traces/v4/enrich_test.go new file mode 100644 index 0000000000..231e6c8a1e --- /dev/null +++ b/pkg/query-service/app/traces/v4/enrich_test.go @@ -0,0 +1,57 @@ +package v4 + +import ( + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +var testColumnName = []struct { + Name string + AttributeKey v3.AttributeKey + ExpectedColumn string +}{ + { + Name: "resource", + AttributeKey: v3.AttributeKey{Key: "collector_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: false}, + ExpectedColumn: "resourceTagsMap['collector_id']", + }, + { + Name: "stringAttribute", + AttributeKey: v3.AttributeKey{Key: "customer_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, + ExpectedColumn: "stringTagMap['customer_id']", + }, + { + Name: "boolAttribute", + AttributeKey: v3.AttributeKey{Key: "has_error", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag, IsColumn: false}, + ExpectedColumn: "boolTagMap['has_error']", + }, + { + Name: "float64Attribute", + AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: false}, + ExpectedColumn: "numberTagMap['count']", + }, + { + Name: "int64Attribute", + AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag, IsColumn: false}, + ExpectedColumn: "numberTagMap['count']", + }, + { + Name: "column", + AttributeKey: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + ExpectedColumn: "name", + }, + { + Name: "missing key", + AttributeKey: v3.AttributeKey{Key: "xyz"}, + ExpectedColumn: "stringTagMap['xyz']", + }, +} + +// func TestColumnName(t *testing.T) { +// for _, tt := range testColumnName { +// tt.AttributeKey = enrichKeyWithMetadata(tt.AttributeKey, map[string]v3.AttributeKey{}) +// Convey("testColumnName", t, func() { +// Column := getColumnName(tt.AttributeKey) +// So(Column, ShouldEqual, tt.ExpectedColumn) +// }) +// } +// } diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go new file mode 100644 index 0000000000..21fc78ade3 --- /dev/null +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -0,0 +1,435 @@ +package v4 + +import ( + "fmt" + "strings" + + "go.signoz.io/signoz/pkg/query-service/app/resource" + tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + "go.signoz.io/signoz/pkg/query-service/constants" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" +) + +type Options struct { + GraphLimitQtype string + PreferRPM bool +} + +var tracesOperatorMappingV3 = map[v3.FilterOperator]string{ + v3.FilterOperatorIn: "IN", + v3.FilterOperatorNotIn: "NOT IN", + v3.FilterOperatorEqual: "=", + v3.FilterOperatorNotEqual: "!=", + v3.FilterOperatorLessThan: "<", + v3.FilterOperatorLessThanOrEq: "<=", + v3.FilterOperatorGreaterThan: ">", + v3.FilterOperatorGreaterThanOrEq: ">=", + v3.FilterOperatorLike: "ILIKE", + v3.FilterOperatorNotLike: "NOT ILIKE", + v3.FilterOperatorRegex: "match(%s, %s)", + v3.FilterOperatorNotRegex: "NOT match(%s, %s)", + v3.FilterOperatorContains: "ILIKE", + v3.FilterOperatorNotContains: "NOT ILIKE", + v3.FilterOperatorExists: "mapContains(%s, '%s')", + v3.FilterOperatorNotExists: "NOT has(%s%s, '%s')", +} + +func getClickHouseTracesColumnType(columnType v3.AttributeKeyType) string { + if columnType == v3.AttributeKeyTypeResource { + return "resources" + } + return "attributes" +} + +func getClickHouseTracesColumnDataType(columnDataType v3.AttributeKeyDataType) string { + if columnDataType == v3.AttributeKeyDataTypeFloat64 || columnDataType == v3.AttributeKeyDataTypeInt64 { + return "number" + } + if columnDataType == v3.AttributeKeyDataTypeBool { + return "bool" + } + return "string" +} + +func getColumnName(key v3.AttributeKey) string { + // TODO (nitya): + // consider routing things like serviceName col to service.name resource attribute for filtering + // consider using static details for some columns + if key.IsColumn { + return "`" + key.Key + "`" + } + keyType := getClickHouseTracesColumnType(key.Type) + keyDType := getClickHouseTracesColumnDataType(key.DataType) + return fmt.Sprintf("%s_%s['%s']", keyType, keyDType, key.Key) +} + +// getSelectLabels returns the select labels for the query based on groupBy and aggregateOperator +func getSelectLabels(groupBy []v3.AttributeKey) string { + var labels []string + for _, tag := range groupBy { + name := getColumnName(tag) + labels = append(labels, fmt.Sprintf(" %s as `%s`", name, tag.Key)) + } + return strings.Join(labels, ",") +} + +func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) { + var conditions []string + + if fs != nil && len(fs.Items) != 0 { + for _, item := range fs.Items { + + // skip if it's a resource attribute + if item.Key.Type == v3.AttributeKeyTypeResource { + continue + } + + val := item.Value + // generate the key + columnName := getColumnName(item.Key) + var fmtVal string + item.Operator = v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator)))) + if item.Operator != v3.FilterOperatorExists && item.Operator != v3.FilterOperatorNotExists { + var err error + val, err = utils.ValidateAndCastValue(val, item.Key.DataType) + if err != nil { + return "", fmt.Errorf("invalid value for key %s: %v", item.Key.Key, err) + } + } + if val != nil { + fmtVal = utils.ClickHouseFormattedValue(val) + } + if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { + switch item.Operator { + case v3.FilterOperatorContains, v3.FilterOperatorNotContains: + val = utils.QuoteEscapedString(fmt.Sprintf("%v", item.Value)) + conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, operator, val)) + case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex: + conditions = append(conditions, fmt.Sprintf(operator, columnName, fmtVal)) + case v3.FilterOperatorExists, v3.FilterOperatorNotExists: + if item.Key.IsColumn { + subQuery, err := tracesV3.ExistsSubQueryForFixedColumn(item.Key, item.Operator) + if err != nil { + return "", err + } + conditions = append(conditions, subQuery) + } else { + cType := getClickHouseTracesColumnType(item.Key.Type) + cDataType := getClickHouseTracesColumnDataType(item.Key.DataType) + col := fmt.Sprintf("%s_%s", cType, cDataType) + conditions = append(conditions, fmt.Sprintf(operator, col, item.Key.Key)) + } + + default: + conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal)) + } + } else { + return "", fmt.Errorf("unsupported operator %s", item.Operator) + } + } + } + queryString := strings.Join(conditions, " AND ") + + return queryString, nil +} + +func handleEmptyValuesInGroupBy(groupBy []v3.AttributeKey) (string, error) { + filterItems := []v3.FilterItem{} + if len(groupBy) != 0 { + for _, item := range groupBy { + if !item.IsColumn { + filterItems = append(filterItems, v3.FilterItem{ + Key: item, + Operator: v3.FilterOperatorExists, + }) + } + } + } + if len(filterItems) != 0 { + filterSet := v3.FilterSet{ + Operator: "AND", + Items: filterItems, + } + return buildTracesFilterQuery(&filterSet) + } + return "", nil +} + +const NANOSECOND = 1000000000 + +// orderBy returns a string of comma separated tags for order by clause +// if there are remaining items which are not present in tags they are also added +// if the order is not specified, it defaults to ASC +func orderBy(panelType v3.PanelType, items []v3.OrderBy, tagLookup map[string]struct{}) []string { + var orderBy []string + + for _, item := range items { + if item.ColumnName == constants.SigNozOrderByValue { + orderBy = append(orderBy, fmt.Sprintf("value %s", item.Order)) + } else if _, ok := tagLookup[item.ColumnName]; ok { + orderBy = append(orderBy, fmt.Sprintf("`%s` %s", item.ColumnName, item.Order)) + } else if panelType == v3.PanelTypeList { + attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn} + name := getColumnName(attr) + if item.IsColumn { + orderBy = append(orderBy, fmt.Sprintf("`%s` %s", name, item.Order)) + } else { + orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) + } + } + } + + return orderBy +} + +func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags []v3.AttributeKey) string { + tagLookup := map[string]struct{}{} + for _, v := range tags { + tagLookup[v.Key] = struct{}{} + } + + orderByArray := orderBy(panelType, items, tagLookup) + + // TODO: check this with logs + if len(orderByArray) == 0 { + if panelType == v3.PanelTypeList { + orderByArray = append(orderByArray, constants.TIMESTAMP+" DESC") + } else if panelType == v3.PanelTypeGraph { + orderByArray = append(orderByArray, "value DESC") + } + } + + str := strings.Join(orderByArray, ",") + return str +} + +func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3.PanelType, options Options) (string, error) { + tracesStart := utils.GetEpochNanoSecs(start) + tracesEnd := utils.GetEpochNanoSecs(end) + + // -1800 this is added so that the bucket start considers all the fingerprints. + bucketStart := tracesStart/NANOSECOND - 1800 + bucketEnd := tracesEnd / NANOSECOND + + timeFilter := fmt.Sprintf("(timestamp >= '%d' AND timestamp <= '%d') AND (ts_bucket_start >= %d AND ts_bucket_start <= %d)", tracesStart, tracesEnd, bucketStart, bucketEnd) + + filterSubQuery, err := buildTracesFilterQuery(mq.Filters) + if err != nil { + return "", err + } + + if filterSubQuery != "" { + filterSubQuery = " AND " + filterSubQuery + } + + resourceSubQuery, err := resource.BuildResourceSubQuery("signoz_logs", "distributed_traces_v3_resource", bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) + if err != nil { + return "", err + } + // join both the filter clauses + if resourceSubQuery != "" { + filterSubQuery = filterSubQuery + " AND (resource_fingerprint GLOBAL IN " + resourceSubQuery + ")" + } + + // timerange will be sent in epoch millisecond + + selectLabels := getSelectLabels(mq.GroupBy) + + orderBy := orderByAttributeKeyTags(panelType, mq.OrderBy, mq.GroupBy) + if orderBy != "" { + orderBy = " order by " + orderBy + } + + if mq.AggregateOperator == v3.AggregateOperatorNoOp { + var query string + if panelType == v3.PanelTypeTrace { + withSubQuery := fmt.Sprintf(constants.TracesExplorerViewSQLSelectWithSubQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_V3_LOCAL_TABLENAME, timeFilter, filterSubQuery) + withSubQuery = tracesV3.AddLimitToQuery(withSubQuery, mq.Limit) + if mq.Offset != 0 { + withSubQuery = tracesV3.AddOffsetToQuery(withSubQuery, mq.Offset) + } + query = fmt.Sprintf(constants.TracesExplorerViewSQLSelectBeforeSubQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_V3) + withSubQuery + ") " + fmt.Sprintf(constants.TracesExplorerViewSQLSelectAfterSubQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_V3, timeFilter) + } else if panelType == v3.PanelTypeList { + if len(mq.SelectColumns) == 0 { + return "", fmt.Errorf("select columns cannot be empty for panelType %s", panelType) + } + selectLabels = getSelectLabels(mq.SelectColumns) + queryNoOpTmpl := fmt.Sprintf("SELECT timestamp as timestamp_datetime, spanID, traceID,%s ", selectLabels) + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_V3 + " where %s %s" + "%s" + query = fmt.Sprintf(queryNoOpTmpl, timeFilter, filterSubQuery, orderBy) + } else { + return "", fmt.Errorf("unsupported aggregate operator %s for panelType %s", mq.AggregateOperator, panelType) + } + return query, nil + // ---- NOOP ends here ---- + } + + having := tracesV3.Having(mq.Having) + if having != "" { + having = " having " + having + } + + groupBy := tracesV3.GroupByAttributeKeyTags(panelType, options.GraphLimitQtype, mq.GroupBy...) + if groupBy != "" { + groupBy = " group by " + groupBy + } + + aggregationKey := "" + if mq.AggregateAttribute.Key != "" { + aggregationKey = getColumnName(mq.AggregateAttribute) + } + + var queryTmpl string + if options.GraphLimitQtype == constants.FirstQueryGraphLimit { + queryTmpl = "SELECT" + } else if panelType == v3.PanelTypeTable { + queryTmpl = + "SELECT " + // step or aggregate interval is whole time period in case of table panel + step = (tracesEnd - tracesStart) / 1000000000 + } else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue { + // Select the aggregate value for interval + queryTmpl = + fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts,", step) + } + + queryTmpl = queryTmpl + selectLabels + + " %s as value " + + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_V3 + + " where " + timeFilter + "%s" + + "%s%s" + + "%s" + + // we don't need value for first query + if options.GraphLimitQtype == constants.FirstQueryGraphLimit { + queryTmpl = "SELECT " + tracesV3.GetSelectKeys(mq.AggregateOperator, mq.GroupBy) + " from (" + queryTmpl + ")" + } + + emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(mq.GroupBy) + if err != nil { + return "", err + } + filterSubQuery += emptyValuesInGroupByFilter + + if options.GraphLimitQtype == constants.SecondQueryGraphLimit { + filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", tracesV3.GetSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "%s)" + } + + switch mq.AggregateOperator { + case v3.AggregateOperatorRateSum, + v3.AggregateOperatorRateMax, + v3.AggregateOperatorRateAvg, + v3.AggregateOperatorRateMin, + v3.AggregateOperatorRate: + + rate := float64(step) + if options.PreferRPM { + rate = rate / 60.0 + } + + op := fmt.Sprintf("%s(%s)/%f", tracesV3.AggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey, rate) + query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) + return query, nil + case + v3.AggregateOperatorP05, + v3.AggregateOperatorP10, + v3.AggregateOperatorP20, + v3.AggregateOperatorP25, + v3.AggregateOperatorP50, + v3.AggregateOperatorP75, + v3.AggregateOperatorP90, + v3.AggregateOperatorP95, + v3.AggregateOperatorP99: + op := fmt.Sprintf("quantile(%v)(%s)", tracesV3.AggregateOperatorToPercentile[mq.AggregateOperator], aggregationKey) + query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) + return query, nil + case v3.AggregateOperatorAvg, v3.AggregateOperatorSum, v3.AggregateOperatorMin, v3.AggregateOperatorMax: + op := fmt.Sprintf("%s(%s)", tracesV3.AggregateOperatorToSQLFunc[mq.AggregateOperator], aggregationKey) + query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) + return query, nil + case v3.AggregateOperatorCount: + if mq.AggregateAttribute.Key != "" { + if mq.AggregateAttribute.IsColumn { + subQuery, err := tracesV3.ExistsSubQueryForFixedColumn(mq.AggregateAttribute, v3.FilterOperatorExists) + if err == nil { + filterSubQuery = fmt.Sprintf("%s AND %s", filterSubQuery, subQuery) + } + } else { + // columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(mq.AggregateAttribute) + column := getColumnName(mq.AggregateAttribute) + filterSubQuery = fmt.Sprintf("%s AND has(%s, '%s')", filterSubQuery, column, mq.AggregateAttribute.Key) + } + } + op := "toFloat64(count())" + query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) + return query, nil + case v3.AggregateOperatorCountDistinct: + op := fmt.Sprintf("toFloat64(count(distinct(%s)))", aggregationKey) + query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) + return query, nil + // case v3.AggregateOperatorNoOp: + + // return query, nil + default: + return "", fmt.Errorf("unsupported aggregate operator %s", mq.AggregateOperator) + } +} + +func enrichOrderBy(items []v3.OrderBy, keys map[string]v3.AttributeKey) []v3.OrderBy { + enrichedItems := []v3.OrderBy{} + for i := 0; i < len(items); i++ { + attributeKey := enrichKeyWithMetadata(v3.AttributeKey{ + Key: items[i].ColumnName, + }, keys) + enrichedItems = append(enrichedItems, v3.OrderBy{ + ColumnName: items[i].ColumnName, + Order: items[i].Order, + Key: attributeKey.Key, + DataType: attributeKey.DataType, + Type: attributeKey.Type, + IsColumn: attributeKey.IsColumn, + }) + } + return enrichedItems +} + +// PrepareTracesQuery returns the query string for traces +// start and end are in epoch millisecond +// step is in seconds +func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) { + // adjust the start and end time to the step interval + start = start - (start % (mq.StepInterval * 1000)) + end = end - (end % (mq.StepInterval * 1000)) + if options.GraphLimitQtype == constants.FirstQueryGraphLimit { + // give me just the group by names + query, err := buildTracesQuery(start, end, mq.StepInterval, mq, panelType, options) + if err != nil { + return "", err + } + query = tracesV3.AddLimitToQuery(query, mq.Limit) + + return query, nil + } else if options.GraphLimitQtype == constants.SecondQueryGraphLimit { + query, err := buildTracesQuery(start, end, mq.StepInterval, mq, panelType, options) + if err != nil { + return "", err + } + return query, nil + } + + query, err := buildTracesQuery(start, end, mq.StepInterval, mq, panelType, options) + if err != nil { + return "", err + } + if panelType == v3.PanelTypeValue { + query, err = tracesV3.ReduceToQuery(query, mq.ReduceTo, mq.AggregateOperator) + } + if panelType == v3.PanelTypeList || panelType == v3.PanelTypeTable { + query = tracesV3.AddLimitToQuery(query, mq.Limit) + + if mq.Offset != 0 { + query = tracesV3.AddOffsetToQuery(query, mq.Offset) + } + } + return query, err +} diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go new file mode 100644 index 0000000000..c5662143e5 --- /dev/null +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -0,0 +1,540 @@ +package v4 + +import ( + "testing" + + "go.signoz.io/signoz/pkg/query-service/constants" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +func Test_getClickHouseTracesColumnType(t *testing.T) { + type args struct { + columnType v3.AttributeKeyType + } + tests := []struct { + name string + args args + want string + }{ + { + name: "tag", + args: args{ + columnType: v3.AttributeKeyTypeTag, + }, + want: "attributes", + }, + { + name: "resource", + args: args{ + columnType: v3.AttributeKeyTypeResource, + }, + want: "resources", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getClickHouseTracesColumnType(tt.args.columnType); got != tt.want { + t.Errorf("GetClickhouseTracesColumnType() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getClickHouseTracesColumnDataType(t *testing.T) { + type args struct { + columnDataType v3.AttributeKeyDataType + } + tests := []struct { + name string + args args + want string + }{ + { + name: "string", + args: args{ + columnDataType: v3.AttributeKeyDataTypeString, + }, + want: "string", + }, + { + name: "float64", + args: args{ + columnDataType: v3.AttributeKeyDataTypeFloat64, + }, + want: "number", + }, + { + name: "int64", + args: args{ + columnDataType: v3.AttributeKeyDataTypeInt64, + }, + want: "number", + }, + { + name: "bool", + args: args{ + columnDataType: v3.AttributeKeyDataTypeBool, + }, + want: "bool", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getClickHouseTracesColumnDataType(tt.args.columnDataType); got != tt.want { + t.Errorf("getClickhouseTracesColumnDataType() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getColumnName(t *testing.T) { + type args struct { + key v3.AttributeKey + } + tests := []struct { + name string + args args + want string + }{ + { + name: "tag", + args: args{ + key: v3.AttributeKey{Key: "data", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + }, + want: "attributes_string['data']", + }, + { + name: "column", + args: args{ + key: v3.AttributeKey{Key: "data", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + want: "`data`", + }, + { + name: "missing meta", + args: args{ + key: v3.AttributeKey{Key: "xyz"}, + }, + want: "attributes_string['xyz']", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getColumnName(tt.args.key); got != tt.want { + t.Errorf("getColumnName() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getSelectLabels(t *testing.T) { + type args struct { + groupBy []v3.AttributeKey + } + tests := []struct { + name string + args args + want string + }{ + { + name: "count", + args: args{ + groupBy: []v3.AttributeKey{{Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + want: " attributes_string['user_name'] as `user_name`", + }, + { + name: "multiple group by", + args: args{ + groupBy: []v3.AttributeKey{ + {Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "service_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: true}, + }, + }, + want: " attributes_string['user_name'] as `user_name`, `service_name` as `service_name`", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getSelectLabels(tt.args.groupBy); got != tt.want { + t.Errorf("getSelectLabels() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_buildTracesFilterQuery(t *testing.T) { + type args struct { + fs *v3.FilterSet + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Test buildTracesFilterQuery in, nin", + args: args{ + fs: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{"GET", "POST"}, Operator: v3.FilterOperatorIn}, + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{"PUT"}, Operator: v3.FilterOperatorNotIn}, + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: []interface{}{"server"}, Operator: v3.FilterOperatorNotIn}, + }}, + }, + want: "attributes_string['method'] IN ['GET','POST'] AND attributes_string['method'] NOT IN ['PUT']", + wantErr: false, + }, + { + name: "Test buildTracesFilterQuery not eq, neq, gt, lt, gte, lte", + args: args{ + fs: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 102, Operator: v3.FilterOperatorEqual}, + {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 100, Operator: v3.FilterOperatorNotEqual}, + {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: v3.FilterOperatorGreaterThan}, + {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 200, Operator: v3.FilterOperatorLessThan}, + {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: v3.FilterOperatorGreaterThanOrEq}, + {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 200, Operator: v3.FilterOperatorLessThanOrEq}, + }}, + }, + want: "attributes_number['duration'] = 102 AND attributes_number['duration'] != 100 AND attributes_number['duration'] > 10 AND attributes_number['duration'] < 200" + + " AND attributes_number['duration'] >= 10 AND attributes_number['duration'] <= 200", + wantErr: false, + }, + { + name: "Test contains, ncontains, like, nlike, regex, nregex", + args: args{ + fs: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.", Operator: v3.FilterOperatorContains}, + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "103", Operator: v3.FilterOperatorNotContains}, + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.", Operator: v3.FilterOperatorLike}, + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102", Operator: v3.FilterOperatorNotLike}, + {Key: v3.AttributeKey{Key: "path", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "/mypath", Operator: v3.FilterOperatorRegex}, + {Key: v3.AttributeKey{Key: "path", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "/health.*", Operator: v3.FilterOperatorNotRegex}, + }}, + }, + want: "attributes_string['host'] ILIKE '%102.%' AND attributes_string['host'] NOT ILIKE '%103%' AND attributes_string['host'] ILIKE '102.' AND attributes_string['host'] NOT ILIKE '102' AND " + + "match(`path`, '/mypath') AND NOT match(`path`, '/health.*')", + }, + { + name: "Test exists, nexists", + args: args{ + fs: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: v3.FilterOperatorExists}, + {Key: v3.AttributeKey{Key: "path", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Operator: v3.FilterOperatorNotExists}, + }}, + }, + want: "mapContains(attributes_string, 'host') AND path = ''", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := buildTracesFilterQuery(tt.args.fs) + if (err != nil) != tt.wantErr { + t.Errorf("buildTracesFilterQuery() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("buildTracesFilterQuery() = %v, want %v", got, tt.want) + } + }) + } +} + +var handleEmptyValuesInGroupByData = []struct { + Name string + GroupBy []v3.AttributeKey + ExpectedFilter string +}{ + { + Name: "String type key", + GroupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + ExpectedFilter: " AND has(stringTagMap, 'bytes')", + }, + { + Name: "fixed column type key", + GroupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, + ExpectedFilter: "", + }, + { + Name: "String, float64 and fixed column type key", + GroupBy: []v3.AttributeKey{ + {Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, + {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + ExpectedFilter: " AND has(stringTagMap, 'bytes') AND has(numberTagMap, 'count')", + }, +} + +func Test_handleEmptyValuesInGroupBy(t *testing.T) { + type args struct { + groupBy []v3.AttributeKey + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Test handleEmptyValuesInGroupBy", + args: args{ + groupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + want: "mapContains(attributes_string, 'bytes')", + wantErr: false, + }, + { + name: "Test handleEmptyValuesInGroupBy", + args: args{ + groupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, + }, + want: "", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := handleEmptyValuesInGroupBy(tt.args.groupBy) + if (err != nil) != tt.wantErr { + t.Errorf("handleEmptyValuesInGroupBy() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("handleEmptyValuesInGroupBy() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_buildTracesQuery(t *testing.T) { + type args struct { + start int64 + end int64 + step int64 + mq *v3.BuilderQuery + panelType v3.PanelType + options Options + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Test buildTracesQuery", + args: args{ + panelType: v3.PanelTypeTable, + start: 1680066360726210000, + end: 1680066458000000000, + step: 1000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorCount, + Filters: &v3.FilterSet{ + Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeInt64}, Value: 100, Operator: ">"}, + {Key: v3.AttributeKey{Key: "service.name", Type: v3.AttributeKeyTypeResource, DataType: v3.AttributeKeyDataTypeString}, Value: "myService", Operator: "="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "host", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeResource}}, + OrderBy: []v3.OrderBy{ + {ColumnName: "host", Order: "ASC"}}, + }, + options: Options{}, + }, + want: "SELECT resources_number['host'] as `host` toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_number['bytes'] > 100 AND " + + "(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_traces_v3_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND " + + "(seen_at_ts_bucket_start <= 1680066458) AND simpleJSONExtractString(labels, 'service.name') = 'myService' AND labels like '%service.name%myService%' AND " + + "( (simpleJSONHas(labels, 'host') AND labels like '%host%') ))) " + + "group by `host` order by `host` ASC", + }, + { + name: "test noop list view", + args: args{ + panelType: v3.PanelTypeList, + start: 1680066360726210000, + end: 1680066458000000000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorNoOp, + Filters: &v3.FilterSet{}, + SelectColumns: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, + }, + }, + want: "SELECT timestamp as timestamp_datetime, spanID, traceID, `name` as `name` from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) order by timestamp DESC", + }, + { + name: "test noop trace view", + args: args{ + panelType: v3.PanelTypeTrace, + start: 1680066360726210000, + end: 1680066458000000000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorNoOp, + Filters: &v3.FilterSet{ + Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + {Key: v3.AttributeKey{Key: "service.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "myService", Operator: "="}, + }, + }, + }, + }, + want: "SELECT subQuery.serviceName, subQuery.name, count() AS span_count, subQuery.durationNano, subQuery.traceID AS traceID FROM signoz_traces.distributed_signoz_index_v3 INNER JOIN " + + "( SELECT * FROM (SELECT traceID, durationNano, serviceName, name FROM signoz_traces.signoz_index_v3 WHERE parentSpanID = '' AND (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND " + + "(ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['method'] = 'GET' AND (resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_traces_v3_resource " + + "WHERE (seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) AND simpleJSONExtractString(labels, 'service.name') = 'myService' AND labels like '%service.name%myService%')) " + + "ORDER BY durationNano DESC LIMIT 1 BY traceID LIMIT 100) AS inner_subquery ) AS subQuery ON signoz_traces.distributed_signoz_index_v3.traceID = subQuery.traceID WHERE (timestamp >= '1680066360726210000' AND " + + "timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) GROUP BY subQuery.traceID, subQuery.durationNano, subQuery.name, subQuery.serviceName ORDER BY " + + "subQuery.durationNano desc LIMIT 1 BY subQuery.traceID;", + }, + { + name: "Test order by value with having", + args: args{ + panelType: v3.PanelTypeTable, + start: 1680066360726210000, + end: 1680066458000000000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorCountDistinct, + Filters: &v3.FilterSet{}, + AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "ASC"}}, + Having: []v3.Having{ + { + ColumnName: "name", + Operator: ">", + Value: 10, + }, + }, + }, + }, + want: "SELECT toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND " + + "(ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) having value > 10 order by value ASC", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := buildTracesQuery(tt.args.start, tt.args.end, tt.args.step, tt.args.mq, tt.args.panelType, tt.args.options) + if (err != nil) != tt.wantErr { + t.Errorf("buildTracesQuery() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("buildTracesQuery() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_orderByAttributeKeyTags(t *testing.T) { + type args struct { + panelType v3.PanelType + items []v3.OrderBy + tags []v3.AttributeKey + } + tests := []struct { + name string + args args + want string + }{ + { + name: "test", + args: args{ + panelType: v3.PanelTypeTrace, + items: []v3.OrderBy{{ColumnName: "name", Order: "ASC"}}, + tags: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + want: "`name` ASC", + }, + { + name: "test", + args: args{ + panelType: v3.PanelTypeList, + items: []v3.OrderBy{{ColumnName: "name", Order: "DESC"}}, + tags: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + want: "`name` DESC", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := orderByAttributeKeyTags(tt.args.panelType, tt.args.items, tt.args.tags); got != tt.want { + t.Errorf("orderByAttributeKeyTags() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPrepareTracesQuery(t *testing.T) { + type args struct { + start int64 + end int64 + panelType v3.PanelType + mq *v3.BuilderQuery + options Options + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "test with limit - first", + args: args{ + start: 1680066360726210000, + end: 1680066458000000000, + panelType: v3.PanelTypeTable, + mq: &v3.BuilderQuery{ + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Filters: &v3.FilterSet{}, + AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + GroupBy: []v3.AttributeKey{{Key: "serviceName", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + Limit: 10, + }, + options: Options{ + GraphLimitQtype: constants.FirstQueryGraphLimit, + }, + }, + want: "SELECT `serviceName` from (SELECT `serviceName` as `serviceName` toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 " + + "where (timestamp >= '1680066360726180000' AND timestamp <= '1680066457999980000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066457) group by `serviceName`) LIMIT 10", + }, + { + name: "test with limit - second", + args: args{ + start: 1680066360726210000, + end: 1680066458000000000, + panelType: v3.PanelTypeTable, + mq: &v3.BuilderQuery{ + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Filters: &v3.FilterSet{}, + AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + GroupBy: []v3.AttributeKey{{Key: "serviceName", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + Limit: 10, + }, + options: Options{ + GraphLimitQtype: constants.SecondQueryGraphLimit, + }, + }, + want: "SELECT `serviceName` as `serviceName` toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 where " + + "(timestamp >= '1680066360726180000' AND timestamp <= '1680066457999980000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066457) AND (`serviceName`) GLOBAL IN (%s) group by `serviceName`", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := PrepareTracesQuery(tt.args.start, tt.args.end, tt.args.panelType, tt.args.mq, tt.args.options) + if (err != nil) != tt.wantErr { + t.Errorf("PrepareTracesQuery() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("PrepareTracesQuery() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/query-service/main.go b/pkg/query-service/main.go index cd6a542220..1f1c54b961 100644 --- a/pkg/query-service/main.go +++ b/pkg/query-service/main.go @@ -39,6 +39,7 @@ func main() { var disableRules bool var useLogsNewSchema bool + var useTraceNewSchema bool // the url used to build link in the alert messages in slack and other systems var ruleRepoURL, cacheConfigPath, fluxInterval string var cluster string @@ -50,6 +51,7 @@ func main() { var dialTimeout time.Duration flag.BoolVar(&useLogsNewSchema, "use-logs-new-schema", false, "use logs_v2 schema for logs") + flag.BoolVar(&useTraceNewSchema, "use-trace-new-schema", false, "use new schema for traces") flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") flag.StringVar(&skipTopLvlOpsPath, "skip-top-level-ops", "", "(config file to skip top level operations)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") @@ -87,6 +89,7 @@ func main() { FluxInterval: fluxInterval, Cluster: cluster, UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } // Read the jwt secret key From 7a011f34605b91a34eeb348d95ad94020767e1df Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 22 Oct 2024 10:20:10 +0530 Subject: [PATCH 02/41] fix: add remaining files --- pkg/query-service/app/traces/v4/utils.go | 216 ++++++++++++++++++++++ pkg/query-service/constants/constants.go | 2 + pkg/query-service/rules/manager.go | 17 +- pkg/query-service/rules/threshold_rule.go | 23 ++- 4 files changed, 243 insertions(+), 15 deletions(-) create mode 100644 pkg/query-service/app/traces/v4/utils.go diff --git a/pkg/query-service/app/traces/v4/utils.go b/pkg/query-service/app/traces/v4/utils.go new file mode 100644 index 0000000000..ae132f6d9e --- /dev/null +++ b/pkg/query-service/app/traces/v4/utils.go @@ -0,0 +1,216 @@ +package v4 + +import ( + "strconv" + + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" + "go.uber.org/zap" +) + +var TracesListViewDefaultSelectedColumns = []v3.AttributeKey{ + { + Key: "serviceName", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + }, + { + Key: "name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + }, + { + Key: "durationNano", + DataType: v3.AttributeKeyDataTypeArrayFloat64, + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + }, + { + Key: "httpMethod", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + }, + { + Key: "responseStatusCode", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + IsColumn: true, + }, +} + +// check if traceId filter is used in traces query and return the list of traceIds +func TraceIdFilterUsedWithEqual(params *v3.QueryRangeParamsV3) (bool, []string) { + compositeQuery := params.CompositeQuery + if compositeQuery == nil { + return false, []string{} + } + var traceIds []string + var traceIdFilterUsed bool + + // Build queries for each builder query + for queryName, query := range compositeQuery.BuilderQueries { + if query.Expression != queryName && query.DataSource != v3.DataSourceTraces { + continue + } + + // check filter attribute + if query.Filters != nil && len(query.Filters.Items) != 0 { + for _, item := range query.Filters.Items { + + if item.Key.Key == "traceID" && (item.Operator == v3.FilterOperatorIn || + item.Operator == v3.FilterOperatorEqual) { + traceIdFilterUsed = true + // validate value + var err error + val := item.Value + val, err = utils.ValidateAndCastValue(val, item.Key.DataType) + if err != nil { + zap.L().Error("invalid value for key", zap.String("key", item.Key.Key), zap.Error(err)) + return false, []string{} + } + if val != nil { + fmtVal := extractFormattedStringValues(val) + traceIds = append(traceIds, fmtVal...) + } + } + } + } + + } + + zap.L().Debug("traceIds", zap.Any("traceIds", traceIds)) + return traceIdFilterUsed, traceIds +} + +func extractFormattedStringValues(v interface{}) []string { + // if it's pointer convert it to a value + v = getPointerValue(v) + + switch x := v.(type) { + case string: + return []string{x} + + case []interface{}: + if len(x) == 0 { + return []string{} + } + switch x[0].(type) { + case string: + values := []string{} + for _, val := range x { + values = append(values, val.(string)) + } + return values + default: + return []string{} + } + default: + return []string{} + } +} + +func getPointerValue(v interface{}) interface{} { + switch x := v.(type) { + case *uint8: + return *x + case *uint16: + return *x + case *uint32: + return *x + case *uint64: + return *x + case *int: + return *x + case *int8: + return *x + case *int16: + return *x + case *int32: + return *x + case *int64: + return *x + case *float32: + return *x + case *float64: + return *x + case *string: + return *x + case *bool: + return *x + case []interface{}: + values := []interface{}{} + for _, val := range x { + values = append(values, getPointerValue(val)) + } + return values + default: + return v + } +} + +func AddTimestampFilters(minTime int64, maxTime int64, params *v3.QueryRangeParamsV3) { + if minTime == 0 && maxTime == 0 { + return + } + + compositeQuery := params.CompositeQuery + if compositeQuery == nil { + return + } + // Build queries for each builder query and apply timestamp filter only if TraceID is present + for queryName, query := range compositeQuery.BuilderQueries { + if query.Expression != queryName && query.DataSource != v3.DataSourceTraces { + continue + } + + addTimeStampFilter := false + + // check filter attribute + if query.Filters != nil && len(query.Filters.Items) != 0 { + for _, item := range query.Filters.Items { + if item.Key.Key == "traceID" && (item.Operator == v3.FilterOperatorIn || + item.Operator == v3.FilterOperatorEqual) { + addTimeStampFilter = true + } + } + } + + // add timestamp filter to query only if traceID filter along with equal/similar operator is used + if addTimeStampFilter { + timeFilters := []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "timestamp", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + Value: strconv.FormatUint(uint64(minTime), 10), + Operator: v3.FilterOperatorGreaterThanOrEq, + }, + { + Key: v3.AttributeKey{ + Key: "timestamp", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + Value: strconv.FormatUint(uint64(maxTime), 10), + Operator: v3.FilterOperatorLessThanOrEq, + }, + } + + // add new timestamp filter to query + if query.Filters == nil { + query.Filters = &v3.FilterSet{ + Items: timeFilters, + } + } else { + query.Filters.Items = append(query.Filters.Items, timeFilters...) + } + } + } +} diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 3a65c03be0..7b9e145af1 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -226,7 +226,9 @@ const ( SIGNOZ_SAMPLES_V4_TABLENAME = "distributed_samples_v4" SIGNOZ_TRACE_DBNAME = "signoz_traces" SIGNOZ_SPAN_INDEX_TABLENAME = "distributed_signoz_index_v2" + SIGNOZ_SPAN_INDEX_V3 = "distributed_signoz_index_v3" SIGNOZ_SPAN_INDEX_LOCAL_TABLENAME = "signoz_index_v2" + SIGNOZ_SPAN_INDEX_V3_LOCAL_TABLENAME = "signoz_index_v3" SIGNOZ_TIMESERIES_v4_LOCAL_TABLENAME = "time_series_v4" SIGNOZ_TIMESERIES_v4_6HRS_LOCAL_TABLENAME = "time_series_v4_6hrs" SIGNOZ_TIMESERIES_v4_1DAY_LOCAL_TABLENAME = "time_series_v4_1day" diff --git a/pkg/query-service/rules/manager.go b/pkg/query-service/rules/manager.go index c41d0bbe50..8869ea1610 100644 --- a/pkg/query-service/rules/manager.go +++ b/pkg/query-service/rules/manager.go @@ -38,7 +38,8 @@ type PrepareTaskOptions struct { ManagerOpts *ManagerOptions NotifyFunc NotifyFunc - UseLogsNewSchema bool + UseLogsNewSchema bool + UseTraceNewSchema bool } const taskNamesuffix = "webAppEditor" @@ -81,7 +82,8 @@ type ManagerOptions struct { PrepareTaskFunc func(opts PrepareTaskOptions) (Task, error) - UseLogsNewSchema bool + UseLogsNewSchema bool + UseTraceNewSchema bool } // The Manager manages recording and alerting rules. @@ -104,7 +106,8 @@ type Manager struct { cache cache.Cache prepareTaskFunc func(opts PrepareTaskOptions) (Task, error) - UseLogsNewSchema bool + UseLogsNewSchema bool + UseTraceNewSchema bool } func defaultOptions(o *ManagerOptions) *ManagerOptions { @@ -140,6 +143,7 @@ func defaultPrepareTaskFunc(opts PrepareTaskOptions) (Task, error) { opts.FF, opts.Reader, opts.UseLogsNewSchema, + opts.UseTraceNewSchema, WithEvalDelay(opts.ManagerOpts.EvalDelay), ) @@ -351,7 +355,8 @@ func (m *Manager) editTask(rule *PostableRule, taskName string) error { ManagerOpts: m.opts, NotifyFunc: m.prepareNotifyFunc(), - UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseTraceNewSchema: m.opts.UseTraceNewSchema, }) if err != nil { @@ -473,7 +478,8 @@ func (m *Manager) addTask(rule *PostableRule, taskName string) error { ManagerOpts: m.opts, NotifyFunc: m.prepareNotifyFunc(), - UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseTraceNewSchema: m.opts.UseTraceNewSchema, }) if err != nil { @@ -817,6 +823,7 @@ func (m *Manager) TestNotification(ctx context.Context, ruleStr string) (int, *m m.featureFlags, m.reader, m.opts.UseLogsNewSchema, + m.opts.UseTraceNewSchema, WithSendAlways(), WithSendUnmatched(), ) diff --git a/pkg/query-service/rules/threshold_rule.go b/pkg/query-service/rules/threshold_rule.go index 72b00e2412..5fb28eea82 100644 --- a/pkg/query-service/rules/threshold_rule.go +++ b/pkg/query-service/rules/threshold_rule.go @@ -58,6 +58,7 @@ func NewThresholdRule( featureFlags interfaces.FeatureLookup, reader interfaces.Reader, useLogsNewSchema bool, + useTraceNewSchema bool, opts ...RuleOption, ) (*ThresholdRule, error) { @@ -74,19 +75,21 @@ func NewThresholdRule( } querierOption := querier.QuerierOptions{ - Reader: reader, - Cache: nil, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FeatureLookup: featureFlags, - UseLogsNewSchema: useLogsNewSchema, + Reader: reader, + Cache: nil, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FeatureLookup: featureFlags, + UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } querierOptsV2 := querierV2.QuerierOptions{ - Reader: reader, - Cache: nil, - KeyGenerator: queryBuilder.NewKeyGenerator(), - FeatureLookup: featureFlags, - UseLogsNewSchema: useLogsNewSchema, + Reader: reader, + Cache: nil, + KeyGenerator: queryBuilder.NewKeyGenerator(), + FeatureLookup: featureFlags, + UseLogsNewSchema: useLogsNewSchema, + UseTraceNewSchema: useTraceNewSchema, } t.querier = querier.NewQuerier(querierOption) From dbf5f8b77a4486c288e93a88e01c271051a85b7d Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 22 Oct 2024 14:06:55 +0530 Subject: [PATCH 03/41] fix: integrate with querier --- pkg/query-service/app/http_handler.go | 8 +++++- .../app/logs/v3/query_builder.go | 2 +- .../app/logs/v3/query_builder_test.go | 24 ++++++++--------- .../app/logs/v4/query_builder.go | 2 +- .../app/logs/v4/query_builder_test.go | 12 ++++----- pkg/query-service/app/querier/helper.go | 12 ++++----- pkg/query-service/app/querier/querier.go | 8 +++++- pkg/query-service/app/querier/v2/helper.go | 12 ++++----- pkg/query-service/app/querier/v2/querier.go | 8 +++++- .../app/queryBuilder/query_builder.go | 19 +++++++------- .../app/traces/v3/query_builder.go | 9 ++----- .../app/traces/v3/query_builder_test.go | 26 +++++++++---------- .../app/traces/v4/query_builder.go | 8 +++--- .../app/traces/v4/query_builder_test.go | 9 +++---- pkg/query-service/model/v3/v3.go | 2 +- .../tests/integration/test_utils.go | 1 + 16 files changed, 87 insertions(+), 75 deletions(-) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index f34e8055f8..64ddf029c1 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -39,6 +39,7 @@ import ( querierV2 "go.signoz.io/signoz/pkg/query-service/app/querier/v2" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/cache" "go.signoz.io/signoz/pkg/query-service/common" @@ -218,9 +219,14 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { logsQueryBuilder = logsv4.PrepareLogsQuery } + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if opts.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } + builderOpts := queryBuilder.QueryBuilderOptions{ BuildMetricQuery: metricsv3.PrepareMetricQuery, - BuildTraceQuery: tracesV3.PrepareTracesQuery, + BuildTraceQuery: tracesQueryBuilder, BuildLogQuery: logsQueryBuilder, } aH.queryBuilder = queryBuilder.NewQueryBuilder(builderOpts, aH.featureFlags) diff --git a/pkg/query-service/app/logs/v3/query_builder.go b/pkg/query-service/app/logs/v3/query_builder.go index 05bd799712..40f019ac91 100644 --- a/pkg/query-service/app/logs/v3/query_builder.go +++ b/pkg/query-service/app/logs/v3/query_builder.go @@ -496,7 +496,7 @@ func IsOrderByTs(orderBy []v3.OrderBy) bool { // PrepareLogsQuery prepares the query for logs // start and end are in epoch millisecond // step is in seconds -func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.LogQBOptions) (string, error) { +func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.QBOptions) (string, error) { // adjust the start and end time to the step interval // NOTE: Disabling this as it's creating confusion between charts and actual data diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index 0eb0c202e5..868afd0015 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -1201,7 +1201,7 @@ var testPrepLogsQueryData = []struct { TableName string AggregateOperator v3.AggregateOperator ExpectedQuery string - Options v3.LogQBOptions + Options v3.QBOptions }{ { Name: "Test TS with limit- first", @@ -1223,7 +1223,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT `method` from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') group by `method` order by value DESC) LIMIT 10", - Options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- first - with order by value", @@ -1246,7 +1246,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT `method` from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') group by `method` order by value ASC) LIMIT 10", - Options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- first - with order by attribute", @@ -1269,7 +1269,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT `method` from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') group by `method` order by `method` ASC) LIMIT 10", - Options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- second", @@ -1291,7 +1291,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') AND (`method`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `method`,ts order by value DESC", - Options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, + Options: v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, { Name: "Test TS with limit- second - with order by", @@ -1314,7 +1314,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') AND (`method`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `method`,ts order by `method` ASC", - Options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, + Options: v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, // Live tail { @@ -1334,7 +1334,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND ", - Options: v3.LogQBOptions{IsLivetailQuery: true}, + Options: v3.QBOptions{IsLivetailQuery: true}, }, { Name: "Live Tail Query with contains", @@ -1353,7 +1353,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where attributes_string_value[indexOf(attributes_string_key, 'method')] ILIKE '%GET%' AND ", - Options: v3.LogQBOptions{IsLivetailQuery: true}, + Options: v3.QBOptions{IsLivetailQuery: true}, }, { Name: "Live Tail Query W/O filter", @@ -1369,7 +1369,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where ", - Options: v3.LogQBOptions{IsLivetailQuery: true}, + Options: v3.QBOptions{IsLivetailQuery: true}, }, { Name: "Table query w/o limit", @@ -1385,7 +1385,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) order by value DESC", - Options: v3.LogQBOptions{}, + Options: v3.QBOptions{}, }, { Name: "Table query with limit", @@ -1402,7 +1402,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) order by value DESC LIMIT 10", - Options: v3.LogQBOptions{}, + Options: v3.QBOptions{}, }, { Name: "Ignore offset if order by is timestamp in list queries", @@ -1488,7 +1488,7 @@ var testPrepLogsQueryLimitOffsetData = []struct { TableName string AggregateOperator v3.AggregateOperator ExpectedQuery string - Options v3.LogQBOptions + Options v3.QBOptions }{ { Name: "Test limit less than pageSize - order by ts", diff --git a/pkg/query-service/app/logs/v4/query_builder.go b/pkg/query-service/app/logs/v4/query_builder.go index 7420c3c091..6b8b554465 100644 --- a/pkg/query-service/app/logs/v4/query_builder.go +++ b/pkg/query-service/app/logs/v4/query_builder.go @@ -492,7 +492,7 @@ func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) { } // PrepareLogsQuery prepares the query for logs -func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.LogQBOptions) (string, error) { +func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.QBOptions) (string, error) { // adjust the start and end time to the step interval // NOTE: Disabling this as it's creating confusion between charts and actual data diff --git a/pkg/query-service/app/logs/v4/query_builder_test.go b/pkg/query-service/app/logs/v4/query_builder_test.go index 9c2b1fd2e5..64ba5c5004 100644 --- a/pkg/query-service/app/logs/v4/query_builder_test.go +++ b/pkg/query-service/app/logs/v4/query_builder_test.go @@ -806,7 +806,7 @@ func TestPrepareLogsQuery(t *testing.T) { queryType v3.QueryType panelType v3.PanelType mq *v3.BuilderQuery - options v3.LogQBOptions + options v3.QBOptions } tests := []struct { name string @@ -875,7 +875,7 @@ func TestPrepareLogsQuery(t *testing.T) { Limit: 10, GroupBy: []v3.AttributeKey{{Key: "user", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, }, - options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + options: v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, want: "SELECT `user` from (SELECT attributes_string['user'] as `user`, toFloat64(count(distinct(attributes_string['name']))) as value from signoz_logs.distributed_logs_v2 " + "where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['method'] = 'GET' " + @@ -904,7 +904,7 @@ func TestPrepareLogsQuery(t *testing.T) { GroupBy: []v3.AttributeKey{{Key: "user", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, Limit: 2, }, - options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, + options: v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, want: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string['user'] as `user`, toFloat64(count(distinct(attributes_string['name']))) as value " + "from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND " + @@ -929,7 +929,7 @@ func TestPrepareLogsQuery(t *testing.T) { }, }, }, - options: v3.LogQBOptions{IsLivetailQuery: true}, + options: v3.QBOptions{IsLivetailQuery: true}, }, want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string " + "from signoz_logs.distributed_logs_v2 where attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND ", @@ -952,7 +952,7 @@ func TestPrepareLogsQuery(t *testing.T) { }, }, }, - options: v3.LogQBOptions{IsLivetailQuery: true}, + options: v3.QBOptions{IsLivetailQuery: true}, }, want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string from " + "signoz_logs.distributed_logs_v2 where attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND " + @@ -972,7 +972,7 @@ func TestPrepareLogsQuery(t *testing.T) { Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, }, - options: v3.LogQBOptions{IsLivetailQuery: true}, + options: v3.QBOptions{IsLivetailQuery: true}, }, want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string " + "from signoz_logs.distributed_logs_v2 where ", diff --git a/pkg/query-service/app/querier/helper.go b/pkg/query-service/app/querier/helper.go index 00b287ce8e..798eb8f0b7 100644 --- a/pkg/query-service/app/querier/helper.go +++ b/pkg/query-service/app/querier/helper.go @@ -45,7 +45,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -56,7 +56,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -71,7 +71,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{PreferRPM: preferRPM}, + v3.QBOptions{PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -167,7 +167,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} @@ -178,7 +178,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} @@ -191,7 +191,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{PreferRPM: preferRPM}, + v3.QBOptions{PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: query, Series: nil} diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index 02234bd037..f6927d859c 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -11,6 +11,8 @@ import ( metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" + "go.signoz.io/signoz/pkg/query-service/common" chErrors "go.signoz.io/signoz/pkg/query-service/errors" "go.signoz.io/signoz/pkg/query-service/querycache" @@ -76,6 +78,10 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { if opts.UseLogsNewSchema { logsQueryBuilder = logsV4.PrepareLogsQuery } + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if opts.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } qc := querycache.NewQueryCache(querycache.WithCache(opts.Cache), querycache.WithFluxInterval(opts.FluxInterval)) @@ -87,7 +93,7 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { fluxInterval: opts.FluxInterval, builder: queryBuilder.NewQueryBuilder(queryBuilder.QueryBuilderOptions{ - BuildTraceQuery: tracesV3.PrepareTracesQuery, + BuildTraceQuery: tracesQueryBuilder, BuildLogQuery: logsQueryBuilder, BuildMetricQuery: metricsV3.PrepareMetricQuery, }, opts.FeatureLookup), diff --git a/pkg/query-service/app/querier/v2/helper.go b/pkg/query-service/app/querier/v2/helper.go index bb41bc8c36..09d6cc2309 100644 --- a/pkg/query-service/app/querier/v2/helper.go +++ b/pkg/query-service/app/querier/v2/helper.go @@ -44,7 +44,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -55,7 +55,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -70,7 +70,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{PreferRPM: preferRPM}, + v3.QBOptions{PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -167,7 +167,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} @@ -178,7 +178,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} @@ -191,7 +191,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{PreferRPM: preferRPM}, + v3.QBOptions{PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: query, Series: nil} diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index a524591c6b..7a217a4410 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -11,6 +11,7 @@ import ( metricsV4 "go.signoz.io/signoz/pkg/query-service/app/metrics/v4" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/common" chErrors "go.signoz.io/signoz/pkg/query-service/errors" "go.signoz.io/signoz/pkg/query-service/querycache" @@ -76,6 +77,11 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { logsQueryBuilder = logsV4.PrepareLogsQuery } + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if opts.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } + qc := querycache.NewQueryCache(querycache.WithCache(opts.Cache), querycache.WithFluxInterval(opts.FluxInterval)) return &querier{ @@ -86,7 +92,7 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { fluxInterval: opts.FluxInterval, builder: queryBuilder.NewQueryBuilder(queryBuilder.QueryBuilderOptions{ - BuildTraceQuery: tracesV3.PrepareTracesQuery, + BuildTraceQuery: tracesQueryBuilder, BuildLogQuery: logsQueryBuilder, BuildMetricQuery: metricsV4.PrepareMetricQuery, }, opts.FeatureLookup), diff --git a/pkg/query-service/app/queryBuilder/query_builder.go b/pkg/query-service/app/queryBuilder/query_builder.go index 879c2d5153..de8db2057a 100644 --- a/pkg/query-service/app/queryBuilder/query_builder.go +++ b/pkg/query-service/app/queryBuilder/query_builder.go @@ -6,7 +6,6 @@ import ( "github.com/SigNoz/govaluate" metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" - tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" "go.signoz.io/signoz/pkg/query-service/cache" "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/interfaces" @@ -42,8 +41,8 @@ var SupportedFunctions = []string{ var EvalFuncs = map[string]govaluate.ExpressionFunction{} -type prepareTracesQueryFunc func(start, end int64, panelType v3.PanelType, bq *v3.BuilderQuery, options tracesV3.Options) (string, error) -type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, options v3.LogQBOptions) (string, error) +type prepareTracesQueryFunc func(start, end int64, panelType v3.PanelType, bq *v3.BuilderQuery, options v3.QBOptions) (string, error) +type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, options v3.QBOptions) (string, error) type prepareMetricQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, options metricsV3.Options) (string, error) type QueryBuilder struct { @@ -161,7 +160,7 @@ func (qb *QueryBuilder) PrepareLiveTailQuery(params *v3.QueryRangeParamsV3) (str } for queryName, query := range compositeQuery.BuilderQueries { if query.Expression == queryName { - queryStr, err = qb.options.BuildLogQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{IsLivetailQuery: true}) + queryStr, err = qb.options.BuildLogQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.QBOptions{IsLivetailQuery: true}) if err != nil { return "", err } @@ -195,12 +194,12 @@ func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3) (map[strin // for ts query with group by and limit form two queries if compositeQuery.PanelType == v3.PanelTypeGraph && query.Limit > 0 && len(query.GroupBy) > 0 { limitQuery, err := qb.options.BuildTraceQuery(start, end, compositeQuery.PanelType, query, - tracesV3.Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } placeholderQuery, err := qb.options.BuildTraceQuery(start, end, compositeQuery.PanelType, - query, tracesV3.Options{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + query, v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } @@ -208,7 +207,7 @@ func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3) (map[strin queries[queryName] = query } else { queryString, err := qb.options.BuildTraceQuery(start, end, compositeQuery.PanelType, - query, tracesV3.Options{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) + query, v3.QBOptions{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) if err != nil { return nil, err } @@ -217,18 +216,18 @@ func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3) (map[strin case v3.DataSourceLogs: // for ts query with limit replace it as it is already formed if compositeQuery.PanelType == v3.PanelTypeGraph && query.Limit > 0 && len(query.GroupBy) > 0 { - limitQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + limitQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } - placeholderQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + placeholderQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } query := fmt.Sprintf(placeholderQuery, limitQuery) queries[queryName] = query } else { - queryString, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) + queryString, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.QBOptions{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) if err != nil { return nil, err } diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index efc48b40b6..6ee4073970 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -10,11 +10,6 @@ import ( "go.signoz.io/signoz/pkg/query-service/utils" ) -type Options struct { - GraphLimitQtype string - PreferRPM bool -} - var AggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ v3.AggregateOperatorP05: 0.05, v3.AggregateOperatorP10: 0.10, @@ -238,7 +233,7 @@ func handleEmptyValuesInGroupBy(groupBy []v3.AttributeKey) (string, error) { return "", nil } -func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, panelType v3.PanelType, options Options) (string, error) { +func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, panelType v3.PanelType, options v3.QBOptions) (string, error) { filterSubQuery, err := buildTracesFilterQuery(mq.Filters) if err != nil { @@ -504,7 +499,7 @@ func AddOffsetToQuery(query string, offset uint64) string { // PrepareTracesQuery returns the query string for traces // start and end are in epoch millisecond // step is in seconds -func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) { +func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.QBOptions) (string, error) { // adjust the start and end time to the step interval start = start - (start % (mq.StepInterval * 1000)) end = end - (end % (mq.StepInterval * 1000)) diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 3903a895dd..23b73a134d 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -490,7 +490,7 @@ var testBuildTracesQueryData = []struct { AggregateOperator v3.AggregateOperator ExpectedQuery string PanelType v3.PanelType - Options Options + Options v3.QBOptions }{ { Name: "Test aggregate count on fixed column of float64 type", @@ -524,7 +524,7 @@ var testBuildTracesQueryData = []struct { " signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <=" + " '1680066458000000000') group by ts order by value DESC", PanelType: v3.PanelTypeGraph, - Options: Options{GraphLimitQtype: "", PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: "", PreferRPM: true}, }, { Name: "Test aggregate count on fixed column of float64 type with filter", @@ -867,7 +867,7 @@ var testBuildTracesQueryData = []struct { "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " AND has(stringTagMap, 'method') group by `method`,ts order by `method` ASC", PanelType: v3.PanelTypeGraph, - Options: Options{GraphLimitQtype: "", + Options: v3.QBOptions{GraphLimitQtype: "", PreferRPM: false, }, }, @@ -892,7 +892,7 @@ var testBuildTracesQueryData = []struct { "AND has(stringTagMap, 'method') group by `method`,ts " + "order by `method` ASC", PanelType: v3.PanelTypeGraph, - Options: Options{GraphLimitQtype: "", PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: "", PreferRPM: true}, }, { Name: "Test aggregate RateSum without fixed column", @@ -916,7 +916,7 @@ var testBuildTracesQueryData = []struct { "AND has(stringTagMap, 'method') group by `method`,ts " + "order by `method` ASC", PanelType: v3.PanelTypeGraph, - Options: Options{GraphLimitQtype: "", PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: "", PreferRPM: true}, }, { Name: "Test aggregate with having clause", @@ -1207,7 +1207,7 @@ var testPrepTracesQueryData = []struct { BuilderQuery *v3.BuilderQuery ExpectedQuery string Keys map[string]v3.AttributeKey - Options Options + Options v3.QBOptions }{ { Name: "Test TS with limit- first", @@ -1232,7 +1232,7 @@ var testPrepTracesQueryData = []struct { " where (timestamp >= '1680066360000000000' AND timestamp <= '1680066420000000000') AND" + " stringTagMap['method'] = 'GET' AND has(stringTagMap, 'method') group by `method` order by value DESC) LIMIT 10", Keys: map[string]v3.AttributeKey{"name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, @@ -1261,7 +1261,7 @@ var testPrepTracesQueryData = []struct { " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') group by `method` order by value ASC) LIMIT 10", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, @@ -1287,7 +1287,7 @@ var testPrepTracesQueryData = []struct { " AND timestamp <= '1680066420000000000') " + "group by `serviceName` order by `serviceName` ASC) LIMIT 10", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, @@ -1317,7 +1317,7 @@ var testPrepTracesQueryData = []struct { " AND timestamp <= '1680066420000000000') AND has(stringTagMap, 'http.method') " + "group by `serviceName`,`http.method` order by `serviceName` ASC,value ASC) LIMIT 10", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, @@ -1345,7 +1345,7 @@ var testPrepTracesQueryData = []struct { " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by value DESC", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, @@ -1373,7 +1373,7 @@ var testPrepTracesQueryData = []struct { " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360000000000'" + " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by `method` ASC", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, @@ -1408,7 +1408,7 @@ var testPrepTracesQueryData = []struct { "AND (`method`,`name`) GLOBAL IN (%s) group by `method`,`name`,ts " + "order by `method` ASC,`name` ASC", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index 21fc78ade3..d6d394829f 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -173,7 +173,7 @@ func orderBy(panelType v3.PanelType, items []v3.OrderBy, tagLookup map[string]st attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn} name := getColumnName(attr) if item.IsColumn { - orderBy = append(orderBy, fmt.Sprintf("`%s` %s", name, item.Order)) + orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) } else { orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) } @@ -204,7 +204,7 @@ func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags [] return str } -func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3.PanelType, options Options) (string, error) { +func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3.PanelType, options v3.QBOptions) (string, error) { tracesStart := utils.GetEpochNanoSecs(start) tracesEnd := utils.GetEpochNanoSecs(end) @@ -223,7 +223,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. filterSubQuery = " AND " + filterSubQuery } - resourceSubQuery, err := resource.BuildResourceSubQuery("signoz_logs", "distributed_traces_v3_resource", bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) + resourceSubQuery, err := resource.BuildResourceSubQuery("signoz_traces", "distributed_traces_v3_resource", bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) if err != nil { return "", err } @@ -396,7 +396,7 @@ func enrichOrderBy(items []v3.OrderBy, keys map[string]v3.AttributeKey) []v3.Ord // PrepareTracesQuery returns the query string for traces // start and end are in epoch millisecond // step is in seconds -func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) { +func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.QBOptions) (string, error) { // adjust the start and end time to the step interval start = start - (start % (mq.StepInterval * 1000)) end = end - (end % (mq.StepInterval * 1000)) diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index c5662143e5..bde9989d33 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -315,7 +315,7 @@ func Test_buildTracesQuery(t *testing.T) { step int64 mq *v3.BuilderQuery panelType v3.PanelType - options Options + options v3.QBOptions } tests := []struct { name string @@ -342,7 +342,6 @@ func Test_buildTracesQuery(t *testing.T) { OrderBy: []v3.OrderBy{ {ColumnName: "host", Order: "ASC"}}, }, - options: Options{}, }, want: "SELECT resources_number['host'] as `host` toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_number['bytes'] > 100 AND " + @@ -473,7 +472,7 @@ func TestPrepareTracesQuery(t *testing.T) { end int64 panelType v3.PanelType mq *v3.BuilderQuery - options Options + options v3.QBOptions } tests := []struct { name string @@ -495,7 +494,7 @@ func TestPrepareTracesQuery(t *testing.T) { GroupBy: []v3.AttributeKey{{Key: "serviceName", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, Limit: 10, }, - options: Options{ + options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, @@ -516,7 +515,7 @@ func TestPrepareTracesQuery(t *testing.T) { GroupBy: []v3.AttributeKey{{Key: "serviceName", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, Limit: 10, }, - options: Options{ + options: v3.QBOptions{ GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index a58575cd08..550fc65f5f 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -1308,7 +1308,7 @@ type URLShareableOptions struct { SelectColumns []AttributeKey `json:"selectColumns"` } -type LogQBOptions struct { +type QBOptions struct { GraphLimitQtype string IsLivetailQuery bool PreferRPM bool diff --git a/pkg/query-service/tests/integration/test_utils.go b/pkg/query-service/tests/integration/test_utils.go index d060433dba..c1c71808a9 100644 --- a/pkg/query-service/tests/integration/test_utils.go +++ b/pkg/query-service/tests/integration/test_utils.go @@ -46,6 +46,7 @@ func NewMockClickhouseReader( featureFlags, "", true, + true, ) return reader, mockDB From 21150938766aca707ac5199537d241f937c2366a Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 22 Oct 2024 22:50:57 +0530 Subject: [PATCH 04/41] fix: get trace by id api updated --- ee/query-service/app/api/traces.go | 45 ++++--- .../app/clickhouseReader/options.go | 9 ++ .../app/clickhouseReader/reader.go | 122 ++++++++++++++++++ pkg/query-service/model/response.go | 17 +++ 4 files changed, 170 insertions(+), 23 deletions(-) diff --git a/ee/query-service/app/api/traces.go b/ee/query-service/app/api/traces.go index 3864fc672e..5c65089cd0 100644 --- a/ee/query-service/app/api/traces.go +++ b/ee/query-service/app/api/traces.go @@ -2,32 +2,31 @@ package api import ( "net/http" - - "go.signoz.io/signoz/ee/query-service/app/db" - "go.signoz.io/signoz/ee/query-service/model" - baseapp "go.signoz.io/signoz/pkg/query-service/app" - basemodel "go.signoz.io/signoz/pkg/query-service/model" - "go.uber.org/zap" ) func (ah *APIHandler) searchTraces(w http.ResponseWriter, r *http.Request) { - if !ah.CheckFeature(basemodel.SmartTraceDetail) { - zap.L().Info("SmartTraceDetail feature is not enabled in this plan") - ah.APIHandler.SearchTraces(w, r) - return - } - searchTracesParams, err := baseapp.ParseSearchTracesParams(r) - if err != nil { - RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params") - return - } - - result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm) - if ah.HandleError(w, err, http.StatusBadRequest) { - return - } - - ah.WriteJSON(w, r, result) + ah.APIHandler.SearchTraces(w, r) + return + + // This is commented since this will be taken care by new trace API + + // if !ah.CheckFeature(basemodel.SmartTraceDetail) { + // zap.L().Info("SmartTraceDetail feature is not enabled in this plan") + // ah.APIHandler.SearchTraces(w, r) + // return + // } + // searchTracesParams, err := baseapp.ParseSearchTracesParams(r) + // if err != nil { + // RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading params") + // return + // } + + // result, err := ah.opts.DataConnector.SearchTraces(r.Context(), searchTracesParams, db.SmartTraceAlgorithm) + // if ah.HandleError(w, err, http.StatusBadRequest) { + // return + // } + + // ah.WriteJSON(w, r, result) } diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index 695bef8570..8b75ab5cc9 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -45,6 +45,9 @@ const ( defaultLogsTableV2 string = "distributed_logs_v2" defaultLogsResourceLocalTableV2 string = "logs_v2_resource" defaultLogsResourceTableV2 string = "distributed_logs_v2_resource" + + defaultTraceIndexTableV3 string = "distributed_signoz_index_v3" + defaultTraceResourceTableV3 string = "distributed_traces_v3_resource" ) // NamespaceConfig is Clickhouse's internal configuration data @@ -82,6 +85,9 @@ type namespaceConfig struct { LogsTableV2 string LogsResourceLocalTableV2 string LogsResourceTableV2 string + + TraceIndexTableV3 string + TraceResourceTableV3 string } // Connecto defines how to connect to the database @@ -174,6 +180,9 @@ func NewOptions( LogsLocalTableV2: defaultLogsLocalTableV2, LogsResourceTableV2: defaultLogsResourceTableV2, LogsResourceLocalTableV2: defaultLogsResourceLocalTableV2, + + TraceIndexTableV3: defaultTraceIndexTableV3, + TraceResourceTableV3: defaultTraceResourceTableV3, }, others: make(map[string]*namespaceConfig, len(otherNamespaces)), } diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index cc2fc2d7f6..63be4652c2 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -149,6 +149,9 @@ type ClickHouseReader struct { useTraceNewSchema bool logsTableName string logsLocalTableName string + + traceIndexTableV3 string + traceResourceTableV3 string } // NewTraceReader returns a TraceReader for the database @@ -257,6 +260,9 @@ func NewReaderFromClickhouseConnection( logsResourceLocalTableV2: options.primary.LogsResourceLocalTableV2, logsTableName: logsTableName, logsLocalTableName: logsLocalTableName, + + traceIndexTableV3: options.primary.TraceIndexTableV3, + traceResourceTableV3: options.primary.TraceResourceTableV3, } } @@ -1670,10 +1676,125 @@ func (r *ClickHouseReader) GetUsage(ctx context.Context, queryParams *model.GetU return &usageItems, nil } +func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.SearchTracesParams) (*[]model.SearchSpansResult, error) { + var countSpans uint64 + // TODO(nitya): check if we can use timestamp filter here + countQuery := fmt.Sprintf("SELECT count() as count from %s.%s WHERE traceID=$1", r.TraceDB, r.traceIndexTableV3) + err := r.db.QueryRow(ctx, countQuery, params.TraceID).Scan(&countSpans) + if err != nil { + zap.L().Error("Error in processing sql query", zap.Error(err)) + return nil, fmt.Errorf("error in processing sql query") + } + + if countSpans > uint64(params.MaxSpansInTrace) { + zap.L().Error("Max spans allowed in a trace limit reached", zap.Int("MaxSpansInTrace", params.MaxSpansInTrace), + zap.Uint64("Count", countSpans)) + userEmail, err := auth.GetEmailFromJwt(ctx) + if err == nil { + data := map[string]interface{}{ + "traceSize": countSpans, + "maxSpansInTraceLimit": params.MaxSpansInTrace, + } + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_MAX_SPANS_ALLOWED_LIMIT_REACHED, data, userEmail, true, false) + } + return nil, fmt.Errorf("max spans allowed in trace limit reached, please contact support for more details") + } + + userEmail, err := auth.GetEmailFromJwt(ctx) + if err == nil { + data := map[string]interface{}{ + "traceSize": countSpans, + } + telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_TRACE_DETAIL_API, data, userEmail, true, false) + } + + var startTime, endTime, durationNano uint64 + var searchScanResponses []model.SearchSpanResponseItemV2 + + query := fmt.Sprintf("SELECT timestamp, durationNano, spanID, traceID, hasError, kind, serviceName, name, references, attributes_string, events, statusMessage, statusCodeString, spanKind FROM %s.%s WHERE traceID=$1", r.TraceDB, r.traceIndexTableV3) + + start := time.Now() + + err = r.db.Select(ctx, &searchScanResponses, query, params.TraceID) + + zap.L().Info(query) + + if err != nil { + zap.L().Error("Error in processing sql query", zap.Error(err)) + return nil, fmt.Errorf("error in processing sql query") + } + end := time.Now() + zap.L().Debug("getTraceSQLQuery took: ", zap.Duration("duration", end.Sub(start))) + searchSpansResult := []model.SearchSpansResult{ + { + Columns: []string{"__time", "SpanId", "TraceId", "ServiceName", "Name", "Kind", "DurationNano", "TagsKeys", "TagsValues", "References", "Events", "HasError", "StatusMessage", "StatusCodeString", "SpanKind"}, + Events: make([][]interface{}, len(searchScanResponses)), + IsSubTree: false, + }, + } + + searchSpanResponses := []model.SearchSpanResponseItem{} + start = time.Now() + for _, item := range searchScanResponses { + ref := []model.OtelSpanRef{} + err := json.Unmarshal([]byte(item.References), &ref) + if err != nil { + zap.L().Error("Error unmarshalling references", zap.Error(err)) + return nil, err + } + + // TODO(nitya): add attributes_number and attributes_bool and resources_string to this + jsonItem := model.SearchSpanResponseItem{ + SpanID: item.SpanID, + TraceID: item.TraceID, + ServiceName: item.ServiceName, + Name: item.Name, + Kind: int32(item.Kind), + DurationNano: int64(item.DurationNano), + HasError: item.HasError, + StatusMessage: item.StatusMessage, + StatusCodeString: item.StatusCodeString, + SpanKind: item.SpanKind, + References: ref, + Events: item.Events, + TagMap: item.TagMap, + } + + jsonItem.TimeUnixNano = uint64(item.TimeUnixNano.UnixNano() / 1000000) + + searchSpanResponses = append(searchSpanResponses, jsonItem) + if startTime == 0 || jsonItem.TimeUnixNano < startTime { + startTime = jsonItem.TimeUnixNano + } + if endTime == 0 || jsonItem.TimeUnixNano > endTime { + endTime = jsonItem.TimeUnixNano + } + if durationNano == 0 || uint64(jsonItem.DurationNano) > durationNano { + durationNano = uint64(jsonItem.DurationNano) + } + } + end = time.Now() + zap.L().Debug("getTraceSQLQuery unmarshal took: ", zap.Duration("duration", end.Sub(start))) + + for i, item := range searchSpanResponses { + spanEvents := item.GetValues() + searchSpansResult[0].Events[i] = spanEvents + } + + searchSpansResult[0].StartTimestampMillis = startTime - (durationNano / 1000000) + searchSpansResult[0].EndTimestampMillis = endTime + (durationNano / 1000000) + + return &searchSpansResult, nil +} + func (r *ClickHouseReader) SearchTraces(ctx context.Context, params *model.SearchTracesParams, smartTraceAlgorithm func(payload []model.SearchSpanResponseItem, targetSpanId string, levelUp int, levelDown int, spanLimit int) ([]model.SearchSpansResult, error)) (*[]model.SearchSpansResult, error) { + if r.useTraceNewSchema { + return r.SearchTracesV2(ctx, params) + } + var countSpans uint64 countQuery := fmt.Sprintf("SELECT count() as count from %s.%s WHERE traceID=$1", r.TraceDB, r.SpansTable) err := r.db.QueryRow(ctx, countQuery, params.TraceID).Scan(&countSpans) @@ -1750,6 +1871,7 @@ func (r *ClickHouseReader) SearchTraces(ctx context.Context, params *model.Searc err = r.featureFlags.CheckFeature(model.SmartTraceDetail) smartAlgoEnabled := err == nil + // TODO(nitya): this will never run remove it if len(searchScanResponses) > params.SpansRenderLimit && smartAlgoEnabled { start = time.Now() searchSpansResult, err = smartTraceAlgorithm(searchSpanResponses, params.SpanID, params.LevelUp, params.LevelDown, params.SpansRenderLimit) diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 3a720aed5e..503770a257 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -269,6 +269,23 @@ type SearchSpanResponseItem struct { SpanKind string `json:"spanKind"` } +type SearchSpanResponseItemV2 struct { + TimeUnixNano time.Time `json:"timestamp" ch:"timestamp"` + DurationNano uint64 `json:"durationNano" ch:"durationNano"` + SpanID string `json:"spanId" ch:"spanID"` + TraceID string `json:"traceId" ch:"traceID"` + HasError bool `json:"hasError" ch:"hasError"` + Kind int8 `json:"kind" ch:"kind"` + ServiceName string `json:"serviceName" ch:"serviceName"` + Name string `json:"name" ch:"name"` + References string `json:"references,omitempty" ch:"references"` + TagMap map[string]string `json:"tagMap" ch:"attributes_string"` + Events []string `json:"event" ch:"events"` + StatusMessage string `json:"statusMessage" ch:"statusMessage"` + StatusCodeString string `json:"statusCodeString" ch:"statusCodeString"` + SpanKind string `json:"spanKind" ch:"spanKind"` +} + type OtelSpanRef struct { TraceId string `json:"traceId,omitempty"` SpanId string `json:"spanId,omitempty"` From 9bc7c8708af674a4f00c83153690dc345bcadd10 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 23 Oct 2024 10:49:35 +0530 Subject: [PATCH 05/41] fix: add servicename resource filter --- pkg/query-service/app/http_handler.go | 7 +++- pkg/query-service/app/traces/v4/enrich.go | 32 +++++++++++++++++++ .../app/traces/v4/query_builder.go | 18 ----------- .../rules/threshold_rule_test.go | 28 ++++++++-------- 4 files changed, 52 insertions(+), 33 deletions(-) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 64ddf029c1..996be49eda 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -4372,7 +4372,12 @@ func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.Que RespondError(w, apiErrObj, errQuriesByName) return } - tracesV3.Enrich(queryRangeParams, spanKeys) + if aH.UseTraceNewSchema { + tracesV4.Enrich(queryRangeParams, spanKeys) + } else { + tracesV3.Enrich(queryRangeParams, spanKeys) + } + } // WARN: Only works for AND operator in traces query diff --git a/pkg/query-service/app/traces/v4/enrich.go b/pkg/query-service/app/traces/v4/enrich.go index 46935aef3a..678913b659 100644 --- a/pkg/query-service/app/traces/v4/enrich.go +++ b/pkg/query-service/app/traces/v4/enrich.go @@ -37,6 +37,20 @@ func EnrichTracesQuery(query *v3.BuilderQuery, keys map[string]v3.AttributeKey) if query.Filters != nil && len(query.Filters.Items) > 0 { for idx, filter := range query.Filters.Items { query.Filters.Items[idx].Key = enrichKeyWithMetadata(filter.Key, keys) + // if the serviceName column is used, use the corresponding resource attribute as well during filtering + if filter.Key.Key == "serviceName" && filter.Key.IsColumn { + query.Filters.Items = append(query.Filters.Items, v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + IsColumn: false, + }, + Operator: filter.Operator, + Value: filter.Value, + }) + } + } } // enrich group by @@ -50,3 +64,21 @@ func EnrichTracesQuery(query *v3.BuilderQuery, keys map[string]v3.AttributeKey) query.SelectColumns[idx] = enrichKeyWithMetadata(selectColumn, keys) } } + +func enrichOrderBy(items []v3.OrderBy, keys map[string]v3.AttributeKey) []v3.OrderBy { + enrichedItems := []v3.OrderBy{} + for i := 0; i < len(items); i++ { + attributeKey := enrichKeyWithMetadata(v3.AttributeKey{ + Key: items[i].ColumnName, + }, keys) + enrichedItems = append(enrichedItems, v3.OrderBy{ + ColumnName: items[i].ColumnName, + Order: items[i].Order, + Key: attributeKey.Key, + DataType: attributeKey.DataType, + Type: attributeKey.Type, + IsColumn: attributeKey.IsColumn, + }) + } + return enrichedItems +} diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index d6d394829f..455de7d8bb 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -375,24 +375,6 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. } } -func enrichOrderBy(items []v3.OrderBy, keys map[string]v3.AttributeKey) []v3.OrderBy { - enrichedItems := []v3.OrderBy{} - for i := 0; i < len(items); i++ { - attributeKey := enrichKeyWithMetadata(v3.AttributeKey{ - Key: items[i].ColumnName, - }, keys) - enrichedItems = append(enrichedItems, v3.OrderBy{ - ColumnName: items[i].ColumnName, - Order: items[i].Order, - Key: attributeKey.Key, - DataType: attributeKey.DataType, - Type: attributeKey.Type, - IsColumn: attributeKey.IsColumn, - }) - } - return enrichedItems -} - // PrepareTracesQuery returns the query string for traces // start and end are in epoch millisecond // step is in seconds diff --git a/pkg/query-service/rules/threshold_rule_test.go b/pkg/query-service/rules/threshold_rule_test.go index d3d84f06a7..7ce7d24445 100644 --- a/pkg/query-service/rules/threshold_rule_test.go +++ b/pkg/query-service/rules/threshold_rule_test.go @@ -791,7 +791,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) { postableRule.RuleCondition.MatchType = MatchType(c.matchType) postableRule.RuleCondition.Target = &c.target - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -880,7 +880,7 @@ func TestPrepareLinksToLogs(t *testing.T) { } fm := featureManager.StartManager() - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -922,7 +922,7 @@ func TestPrepareLinksToTraces(t *testing.T) { } fm := featureManager.StartManager() - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -998,7 +998,7 @@ func TestThresholdRuleLabelNormalization(t *testing.T) { postableRule.RuleCondition.MatchType = MatchType(c.matchType) postableRule.RuleCondition.Target = &c.target - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -1051,7 +1051,7 @@ func TestThresholdRuleEvalDelay(t *testing.T) { fm := featureManager.StartManager() for idx, c := range cases { - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true) // no eval delay + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true) // no eval delay if err != nil { assert.NoError(t, err) } @@ -1100,7 +1100,7 @@ func TestThresholdRuleClickHouseTmpl(t *testing.T) { fm := featureManager.StartManager() for idx, c := range cases { - rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, WithEvalDelay(2*time.Minute)) + rule, err := NewThresholdRule("69", &postableRule, fm, nil, true, true, WithEvalDelay(2*time.Minute)) if err != nil { assert.NoError(t, err) } @@ -1241,9 +1241,9 @@ func TestThresholdRuleUnitCombinations(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true, true) - rule, err := NewThresholdRule("69", &postableRule, fm, reader, true) + rule, err := NewThresholdRule("69", &postableRule, fm, reader, true, true) rule.TemporalityMap = map[string]map[v3.Temporality]bool{ "signoz_calls_total": { v3.Delta: true, @@ -1340,9 +1340,9 @@ func TestThresholdRuleNoData(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true, true) - rule, err := NewThresholdRule("69", &postableRule, fm, reader, true) + rule, err := NewThresholdRule("69", &postableRule, fm, reader, true, true) rule.TemporalityMap = map[string]map[v3.Temporality]bool{ "signoz_calls_total": { v3.Delta: true, @@ -1445,9 +1445,9 @@ func TestThresholdRuleTracesLink(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true, true) - rule, err := NewThresholdRule("69", &postableRule, fm, reader, true) + rule, err := NewThresholdRule("69", &postableRule, fm, reader, true, true) rule.TemporalityMap = map[string]map[v3.Temporality]bool{ "signoz_calls_total": { v3.Delta: true, @@ -1570,9 +1570,9 @@ func TestThresholdRuleLogsLink(t *testing.T) { } options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") - reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true) + reader := clickhouseReader.NewReaderFromClickhouseConnection(mock, options, nil, "", fm, "", true, true) - rule, err := NewThresholdRule("69", &postableRule, fm, reader, true) + rule, err := NewThresholdRule("69", &postableRule, fm, reader, true, true) rule.TemporalityMap = map[string]map[v3.Temporality]bool{ "signoz_calls_total": { v3.Delta: true, From fd09f57f76d7694702c42a5257f9dcf69d5eec7c Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 23 Oct 2024 17:44:18 +0530 Subject: [PATCH 06/41] fix: tests --- pkg/query-service/app/traces/v4/query_builder_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index bde9989d33..2c0956fdfb 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -345,7 +345,7 @@ func Test_buildTracesQuery(t *testing.T) { }, want: "SELECT resources_number['host'] as `host` toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_number['bytes'] > 100 AND " + - "(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_traces_v3_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND " + + "(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND " + "(seen_at_ts_bucket_start <= 1680066458) AND simpleJSONExtractString(labels, 'service.name') = 'myService' AND labels like '%service.name%myService%' AND " + "( (simpleJSONHas(labels, 'host') AND labels like '%host%') ))) " + "group by `host` order by `host` ASC", @@ -383,7 +383,7 @@ func Test_buildTracesQuery(t *testing.T) { }, want: "SELECT subQuery.serviceName, subQuery.name, count() AS span_count, subQuery.durationNano, subQuery.traceID AS traceID FROM signoz_traces.distributed_signoz_index_v3 INNER JOIN " + "( SELECT * FROM (SELECT traceID, durationNano, serviceName, name FROM signoz_traces.signoz_index_v3 WHERE parentSpanID = '' AND (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND " + - "(ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['method'] = 'GET' AND (resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_logs.distributed_traces_v3_resource " + + "(ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['method'] = 'GET' AND (resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource " + "WHERE (seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) AND simpleJSONExtractString(labels, 'service.name') = 'myService' AND labels like '%service.name%myService%')) " + "ORDER BY durationNano DESC LIMIT 1 BY traceID LIMIT 100) AS inner_subquery ) AS subQuery ON signoz_traces.distributed_signoz_index_v3.traceID = subQuery.traceID WHERE (timestamp >= '1680066360726210000' AND " + "timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) GROUP BY subQuery.traceID, subQuery.durationNano, subQuery.name, subQuery.serviceName ORDER BY " + From 1487820750ed330fb7c07ceb4d029efe721963f0 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 23 Oct 2024 18:06:25 +0530 Subject: [PATCH 07/41] fix: use correct prepQUery --- pkg/query-service/app/querier/helper.go | 12 +++++++++--- pkg/query-service/app/querier/v2/helper.go | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/query-service/app/querier/helper.go b/pkg/query-service/app/querier/helper.go index 798eb8f0b7..1b2acbab8b 100644 --- a/pkg/query-service/app/querier/helper.go +++ b/pkg/query-service/app/querier/helper.go @@ -10,6 +10,7 @@ import ( logsV4 "go.signoz.io/signoz/pkg/query-service/app/logs/v4" metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/common" "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" @@ -158,11 +159,16 @@ func (q *querier) runBuilderQuery( if builderQuery.DataSource == v3.DataSourceTraces { + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if q.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } + var query string var err error // for ts query with group by and limit form two queries if params.CompositeQuery.PanelType == v3.PanelTypeGraph && builderQuery.Limit > 0 && len(builderQuery.GroupBy) > 0 { - limitQuery, err := tracesV3.PrepareTracesQuery( + limitQuery, err := tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, @@ -173,7 +179,7 @@ func (q *querier) runBuilderQuery( ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} return } - placeholderQuery, err := tracesV3.PrepareTracesQuery( + placeholderQuery, err := tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, @@ -186,7 +192,7 @@ func (q *querier) runBuilderQuery( } query = fmt.Sprintf(placeholderQuery, limitQuery) } else { - query, err = tracesV3.PrepareTracesQuery( + query, err = tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, diff --git a/pkg/query-service/app/querier/v2/helper.go b/pkg/query-service/app/querier/v2/helper.go index 09d6cc2309..b62fffe106 100644 --- a/pkg/query-service/app/querier/v2/helper.go +++ b/pkg/query-service/app/querier/v2/helper.go @@ -11,6 +11,7 @@ import ( metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" metricsV4 "go.signoz.io/signoz/pkg/query-service/app/metrics/v4" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/common" "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" @@ -158,11 +159,16 @@ func (q *querier) runBuilderQuery( if builderQuery.DataSource == v3.DataSourceTraces { + tracesQueryBuilder := tracesV3.PrepareTracesQuery + if q.UseTraceNewSchema { + tracesQueryBuilder = tracesV4.PrepareTracesQuery + } + var query string var err error // for ts query with group by and limit form two queries if params.CompositeQuery.PanelType == v3.PanelTypeGraph && builderQuery.Limit > 0 && len(builderQuery.GroupBy) > 0 { - limitQuery, err := tracesV3.PrepareTracesQuery( + limitQuery, err := tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, @@ -173,7 +179,7 @@ func (q *querier) runBuilderQuery( ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} return } - placeholderQuery, err := tracesV3.PrepareTracesQuery( + placeholderQuery, err := tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, @@ -186,7 +192,7 @@ func (q *querier) runBuilderQuery( } query = fmt.Sprintf(placeholderQuery, limitQuery) } else { - query, err = tracesV3.PrepareTracesQuery( + query, err = tracesQueryBuilder( start, end, params.CompositeQuery.PanelType, From 4749ec18bc48666cc398e10bfea3a5bf501092dd Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 23 Oct 2024 23:59:20 +0530 Subject: [PATCH 08/41] fix: services page --- .../app/clickhouseReader/reader.go | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 63be4652c2..217a8837fe 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -473,9 +473,13 @@ func (r *ClickHouseReader) GetQueryRangeResult(ctx context.Context, query *model } func (r *ClickHouseReader) GetServicesList(ctx context.Context) (*[]string, error) { + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } services := []string{} - query := fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.indexTable) + query := fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE ts_bucket_start > (toUnixTimestamp(now()) - 1800) AND toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, indexTable) rows, err := r.db.Query(ctx, query) @@ -537,6 +541,10 @@ func (r *ClickHouseReader) GetTopLevelOperations(ctx context.Context, skipConfig } func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.GetServicesParams, skipConfig *model.SkipConfig) (*[]model.ServiceItem, *model.ApiError) { + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } if r.indexTable == "" { return nil, &model.ApiError{Typ: model.ErrorExec, Err: ErrNoIndexTable} @@ -584,14 +592,14 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G count(*) as numCalls FROM %s.%s WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end`, - r.TraceDB, r.indexTable, + r.TraceDB, indexTable, ) errorQuery := fmt.Sprintf( `SELECT count(*) as numErrors FROM %s.%s WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end AND statusCode=2`, - r.TraceDB, r.indexTable, + r.TraceDB, indexTable, ) args := []interface{}{} @@ -601,6 +609,17 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G clickhouse.Named("serviceName", svc), clickhouse.Named("names", ops), ) + + if r.useTraceNewSchema { + bFilter := " AND ts_bucket_start >= @start_bucket AND ts_bucket_start <= @end_bucket" + query += bFilter + errorQuery += bFilter + args = append(args, + clickhouse.Named("start_bucket", strconv.FormatInt(queryParams.Start.Unix()-1800, 10)), + clickhouse.Named("end_bucket", strconv.FormatInt(queryParams.End.Unix(), 10)), + ) + } + // create TagQuery from TagQueryParams tags := createTagQueryFromTagQueryParams(queryParams.Tags) subQuery, argsSubQuery, errStatus := buildQueryWithTagParams(ctx, tags) From 16a7717598aa53516517c4b33c9b60466568c384 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 24 Oct 2024 12:29:29 +0530 Subject: [PATCH 09/41] fix: minor fixes to use the new table in api's and querier --- .../app/clickhouseReader/reader.go | 86 +++++++++++++------ pkg/query-service/app/querier/v2/querier.go | 9 +- 2 files changed, 67 insertions(+), 28 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 217a8837fe..fc5f3d03b1 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -674,6 +674,11 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams *model.GetServiceOverviewParams, skipConfig *model.SkipConfig) (*[]model.ServiceOverviewItem, *model.ApiError) { + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } + topLevelOps, apiErr := r.GetTopLevelOperations(ctx, skipConfig, *queryParams.Start, *queryParams.End, nil) if apiErr != nil { return nil, apiErr @@ -702,7 +707,7 @@ func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams * count(*) as numCalls FROM %s.%s WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end`, - r.TraceDB, r.indexTable, + r.TraceDB, indexTable, ) args := []interface{}{} args = append(args, namedArgs...) @@ -733,7 +738,7 @@ func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams * count(*) as numErrors FROM %s.%s WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end AND statusCode=2`, - r.TraceDB, r.indexTable, + r.TraceDB, indexTable, ) args = []interface{}{} args = append(args, namedArgs...) @@ -797,6 +802,11 @@ func buildFilterArrayQuery(_ context.Context, excludeMap map[string]struct{}, pa func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *model.SpanFilterParams) (*model.SpanFiltersResponse, *model.ApiError) { + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } + var query string excludeMap := make(map[string]struct{}) for _, e := range queryParams.Exclude { @@ -870,7 +880,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode case constants.TraceID: continue case constants.ServiceName: - finalQuery := fmt.Sprintf("SELECT serviceName, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf("SELECT serviceName, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) finalQuery += query finalQuery += " GROUP BY serviceName" var dBResponse []model.DBResponseServiceName @@ -887,7 +897,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.HttpRoute: - finalQuery := fmt.Sprintf("SELECT httpRoute, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf("SELECT httpRoute, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) finalQuery += query finalQuery += " GROUP BY httpRoute" var dBResponse []model.DBResponseHttpRoute @@ -904,7 +914,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.HttpUrl: - finalQuery := fmt.Sprintf("SELECT httpUrl, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf("SELECT httpUrl, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) finalQuery += query finalQuery += " GROUP BY httpUrl" var dBResponse []model.DBResponseHttpUrl @@ -921,7 +931,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.HttpMethod: - finalQuery := fmt.Sprintf("SELECT httpMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf("SELECT httpMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) finalQuery += query finalQuery += " GROUP BY httpMethod" var dBResponse []model.DBResponseHttpMethod @@ -938,7 +948,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.HttpHost: - finalQuery := fmt.Sprintf("SELECT httpHost, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf("SELECT httpHost, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) finalQuery += query finalQuery += " GROUP BY httpHost" var dBResponse []model.DBResponseHttpHost @@ -955,7 +965,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.OperationRequest: - finalQuery := fmt.Sprintf("SELECT name, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf("SELECT name, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) finalQuery += query finalQuery += " GROUP BY name" var dBResponse []model.DBResponseOperation @@ -972,7 +982,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.Status: - finalQuery := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = true", r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = true", r.TraceDB, indexTable) finalQuery += query var dBResponse []model.DBResponseTotal err := r.db.Select(ctx, &dBResponse, finalQuery, args...) @@ -983,7 +993,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} } - finalQuery2 := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = false", r.TraceDB, r.indexTable) + finalQuery2 := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = false", r.TraceDB, indexTable) finalQuery2 += query var dBResponse2 []model.DBResponseTotal err = r.db.Select(ctx, &dBResponse2, finalQuery2, args...) @@ -1008,7 +1018,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode finalQuery := "" if !durationSortEnabled { // if duration sort is not enabled, we need to get the min and max duration from the index table - finalQuery = fmt.Sprintf("SELECT min(durationNano) as min, max(durationNano) as max FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery = fmt.Sprintf("SELECT min(durationNano) as min, max(durationNano) as max FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) finalQuery += query var dBResponse []model.DBResponseMinMax err = r.db.Select(ctx, &dBResponse, finalQuery, args...) @@ -1053,7 +1063,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.RPCMethod: - finalQuery := fmt.Sprintf("SELECT rpcMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf("SELECT rpcMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) finalQuery += query finalQuery += " GROUP BY rpcMethod" var dBResponse []model.DBResponseRPCMethod @@ -1071,7 +1081,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } case constants.ResponseStatusCode: - finalQuery := fmt.Sprintf("SELECT responseStatusCode, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf("SELECT responseStatusCode, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) finalQuery += query finalQuery += " GROUP BY responseStatusCode" var dBResponse []model.DBResponseStatusCodeMethod @@ -1119,7 +1129,12 @@ func getStatusFilters(query string, statusParams []string, excludeMap map[string func (r *ClickHouseReader) GetFilteredSpans(ctx context.Context, queryParams *model.GetFilteredSpansParams) (*model.GetFilterSpansResponse, *model.ApiError) { - queryTable := fmt.Sprintf("%s.%s", r.TraceDB, r.indexTable) + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } + + queryTable := fmt.Sprintf("%s.%s", r.TraceDB, indexTable) excludeMap := make(map[string]struct{}) for _, e := range queryParams.Exclude { @@ -1410,6 +1425,11 @@ func addExistsOperator(item model.TagQuery, tagMapType string, not bool) (string func (r *ClickHouseReader) GetTagFilters(ctx context.Context, queryParams *model.TagFilterParams) (*model.TagFilters, *model.ApiError) { + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } + excludeMap := make(map[string]struct{}) for _, e := range queryParams.Exclude { if e == constants.OperationRequest { @@ -1465,8 +1485,8 @@ func (r *ClickHouseReader) GetTagFilters(ctx context.Context, queryParams *model tagFilters := []model.TagFilters{} - // Alternative finalQuery := fmt.Sprintf(`SELECT DISTINCT arrayJoin(tagMap.keys) as tagKeys FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, r.indexTable) - finalQuery := fmt.Sprintf(`SELECT groupUniqArrayArray(mapKeys(stringTagMap)) as stringTagKeys, groupUniqArrayArray(mapKeys(numberTagMap)) as numberTagKeys, groupUniqArrayArray(mapKeys(boolTagMap)) as boolTagKeys FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, r.indexTable) + // Alternative finalQuery := fmt.Sprintf(`SELECT DISTINCT arrayJoin(tagMap.keys) as tagKeys FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, indexTable) + finalQuery := fmt.Sprintf(`SELECT groupUniqArrayArray(mapKeys(stringTagMap)) as stringTagKeys, groupUniqArrayArray(mapKeys(numberTagMap)) as numberTagKeys, groupUniqArrayArray(mapKeys(boolTagMap)) as boolTagKeys FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, indexTable) finalQuery += query err := r.db.Select(ctx, &tagFilters, finalQuery, args...) @@ -1514,6 +1534,11 @@ func excludeTags(_ context.Context, tags []string) []string { func (r *ClickHouseReader) GetTagValues(ctx context.Context, queryParams *model.TagFilterParams) (*model.TagValues, *model.ApiError) { + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } + if queryParams.TagKey.Type == model.TagTypeNumber { return &model.TagValues{ NumberTagValues: make([]float64, 0), @@ -1577,7 +1602,7 @@ func (r *ClickHouseReader) GetTagValues(ctx context.Context, queryParams *model. tagValues := []model.TagValues{} - finalQuery := fmt.Sprintf(`SELECT groupArray(DISTINCT stringTagMap[@key]) as stringTagValues FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, r.indexTable) + finalQuery := fmt.Sprintf(`SELECT groupArray(DISTINCT attributes_string[@key]) as stringTagValues FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, indexTable) finalQuery += query finalQuery += " LIMIT @limit" @@ -1609,6 +1634,10 @@ func (r *ClickHouseReader) GetTagValues(ctx context.Context, queryParams *model. } func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *model.GetTopOperationsParams) (*[]model.TopOperationsItem, *model.ApiError) { + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } namedArgs := []interface{}{ clickhouse.Named("start", strconv.FormatInt(queryParams.Start.UnixNano(), 10)), @@ -1628,7 +1657,7 @@ func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *mo name FROM %s.%s WHERE serviceName = @serviceName AND timestamp>= @start AND timestamp<= @end`, - r.TraceDB, r.indexTable, + r.TraceDB, indexTable, ) args := []interface{}{} args = append(args, namedArgs...) @@ -1970,6 +1999,11 @@ func (r *ClickHouseReader) GetDependencyGraph(ctx context.Context, queryParams * func (r *ClickHouseReader) GetFilteredSpansAggregates(ctx context.Context, queryParams *model.GetFilteredSpanAggregatesParams) (*model.GetFilteredSpansAggregatesResponse, *model.ApiError) { + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } + excludeMap := make(map[string]struct{}) for _, e := range queryParams.Exclude { if e == constants.OperationRequest { @@ -2015,7 +2049,7 @@ func (r *ClickHouseReader) GetFilteredSpansAggregates(ctx context.Context, query // Using %s for groupBy params as it can be a custom column and custom columns are not supported by clickhouse-go yet: // issue link: https://github.com/ClickHouse/clickhouse-go/issues/870 if queryParams.GroupBy != "" && columnExists { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, queryParams.GroupBy, aggregation_query, r.TraceDB, r.indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, queryParams.GroupBy, aggregation_query, r.TraceDB, indexTable) args = append(args, clickhouse.Named("groupByVar", queryParams.GroupBy)) } else if queryParams.GroupBy != "" { customStr = strings.Split(queryParams.GroupBy, ".(") @@ -2023,17 +2057,17 @@ func (r *ClickHouseReader) GetFilteredSpansAggregates(ctx context.Context, query return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("GroupBy: %s not supported", queryParams.GroupBy)} } if customStr[1] == string(model.TagTypeString)+")" { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, stringTagMap['%s'] as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, r.indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, stringTagMap['%s'] as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, indexTable) } else if customStr[1] == string(model.TagTypeNumber)+")" { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(numberTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, r.indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(numberTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, indexTable) } else if customStr[1] == string(model.TagTypeBool)+")" { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(boolTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, r.indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(boolTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, indexTable) } else { // return error for unsupported group by return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("GroupBy: %s not supported", queryParams.GroupBy)} } } else { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, aggregation_query, r.TraceDB, r.indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, aggregation_query, r.TraceDB, indexTable) } if len(queryParams.TraceID) > 0 { @@ -3201,11 +3235,15 @@ func (r *ClickHouseReader) GetLogsInfoInLastHeartBeatInterval(ctx context.Contex } func (r *ClickHouseReader) GetTagsInfoInLastHeartBeatInterval(ctx context.Context, interval time.Duration) (*model.TagsInfo, error) { + indexTable := r.indexTable + if r.useTraceNewSchema { + indexTable = r.traceIndexTableV3 + } queryStr := fmt.Sprintf(`select serviceName, stringTagMap['deployment.environment'] as env, stringTagMap['telemetry.sdk.language'] as language from %s.%s where timestamp > toUnixTimestamp(now()-toIntervalMinute(%d)) - group by serviceName, env, language;`, r.TraceDB, r.indexTable, int(interval.Minutes())) + group by serviceName, env, language;`, r.TraceDB, indexTable, int(interval.Minutes())) tagTelemetryDataList := []model.TagTelemetryData{} err := r.db.Select(ctx, &tagTelemetryDataList, queryStr) diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 7a217a4410..70c9f40ac7 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -98,10 +98,11 @@ func NewQuerier(opts QuerierOptions) interfaces.Querier { }, opts.FeatureLookup), featureLookUp: opts.FeatureLookup, - testingMode: opts.TestingMode, - returnedSeries: opts.ReturnedSeries, - returnedErr: opts.ReturnedErr, - UseLogsNewSchema: opts.UseLogsNewSchema, + testingMode: opts.TestingMode, + returnedSeries: opts.ReturnedSeries, + returnedErr: opts.ReturnedErr, + UseLogsNewSchema: opts.UseLogsNewSchema, + UseTraceNewSchema: opts.UseTraceNewSchema, } } From bb41435a2001df75546240497a42ce86ba190da1 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 25 Oct 2024 13:06:54 +0530 Subject: [PATCH 10/41] fix: add support for window based pagination --- pkg/query-service/app/querier/querier.go | 16 ++++++++++------ pkg/query-service/app/querier/v2/querier.go | 16 ++++++++++------ pkg/query-service/app/traces/v4/query_builder.go | 2 ++ .../app/traces/v4/query_builder_test.go | 2 +- pkg/query-service/utils/logs.go | 2 ++ pkg/query-service/utils/logs_test.go | 4 +++- 6 files changed, 28 insertions(+), 14 deletions(-) diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index f6927d859c..6689f85241 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -317,7 +317,7 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } -func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { +func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { res := make([]*v3.Result, 0) qName := "" pageSize := uint64(0) @@ -378,14 +378,18 @@ func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangePar func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3) ([]*v3.Result, map[string]error, error) { // List query has support for only one query. - if q.UseLogsNewSchema && params.CompositeQuery != nil && len(params.CompositeQuery.BuilderQueries) == 1 { + if q.UseLogsNewSchema && params.CompositeQuery != nil && + len(params.CompositeQuery.BuilderQueries) == 1 && + params.CompositeQuery.PanelType != v3.PanelTypeTrace { for _, v := range params.CompositeQuery.BuilderQueries { // only allow of logs queries with timestamp ordering desc - if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { + // TODO(nitya): allow for timestamp asc + if (v.DataSource == v3.DataSourceLogs || v.DataSource == v3.DataSourceTraces) && + len(v.OrderBy) == 1 && + v.OrderBy[0].ColumnName == "timestamp" && + v.OrderBy[0].Order == "desc" { startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) - if len(startEndArr) > 0 { - return q.runLogsListQuery(ctx, params, startEndArr) - } + return q.runWindowBasedListQuery(ctx, params, startEndArr) } } } diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 70c9f40ac7..a373ae1589 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -317,7 +317,7 @@ func (q *querier) runClickHouseQueries(ctx context.Context, params *v3.QueryRang return results, errQueriesByName, err } -func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { +func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryRangeParamsV3, tsRanges []utils.LogsListTsRange) ([]*v3.Result, map[string]error, error) { res := make([]*v3.Result, 0) qName := "" pageSize := uint64(0) @@ -378,14 +378,18 @@ func (q *querier) runLogsListQuery(ctx context.Context, params *v3.QueryRangePar func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3) ([]*v3.Result, map[string]error, error) { // List query has support for only one query. - if q.UseLogsNewSchema && params.CompositeQuery != nil && len(params.CompositeQuery.BuilderQueries) == 1 { + if q.UseLogsNewSchema && params.CompositeQuery != nil && + len(params.CompositeQuery.BuilderQueries) == 1 && + params.CompositeQuery.PanelType != v3.PanelTypeTrace { for _, v := range params.CompositeQuery.BuilderQueries { // only allow of logs queries with timestamp ordering desc - if v.DataSource == v3.DataSourceLogs && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { + // TODO(nitya): allow for timestamp asc + if (v.DataSource == v3.DataSourceLogs || v.DataSource == v3.DataSourceTraces) && + len(v.OrderBy) == 1 && + v.OrderBy[0].ColumnName == "timestamp" && + v.OrderBy[0].Order == "desc" { startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) - if len(startEndArr) > 0 { - return q.runLogsListQuery(ctx, params, startEndArr) - } + return q.runWindowBasedListQuery(ctx, params, startEndArr) } } } diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index 455de7d8bb..aba2bc167f 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -254,6 +254,8 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. if len(mq.SelectColumns) == 0 { return "", fmt.Errorf("select columns cannot be empty for panelType %s", panelType) } + // add it to the select labels + mq.SelectColumns = append(mq.SelectColumns, v3.AttributeKey{Key: "id", IsColumn: true}) selectLabels = getSelectLabels(mq.SelectColumns) queryNoOpTmpl := fmt.Sprintf("SELECT timestamp as timestamp_datetime, spanID, traceID,%s ", selectLabels) + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_V3 + " where %s %s" + "%s" query = fmt.Sprintf(queryNoOpTmpl, timeFilter, filterSubQuery, orderBy) diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index 2c0956fdfb..fa0f1a84ec 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -362,7 +362,7 @@ func Test_buildTracesQuery(t *testing.T) { SelectColumns: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, }, }, - want: "SELECT timestamp as timestamp_datetime, spanID, traceID, `name` as `name` from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + want: "SELECT timestamp as timestamp_datetime, spanID, traceID, `name` as `name`, `id` as `id` from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) order by timestamp DESC", }, { diff --git a/pkg/query-service/utils/logs.go b/pkg/query-service/utils/logs.go index 8efa026b52..b7bf8d9491 100644 --- a/pkg/query-service/utils/logs.go +++ b/pkg/query-service/utils/logs.go @@ -35,6 +35,8 @@ func GetLogsListTsRanges(start, end int64) []LogsListTsRange { tStartNano = startNano } } + } else { + result = append(result, LogsListTsRange{Start: startNano, End: endNano}) } return result } diff --git a/pkg/query-service/utils/logs_test.go b/pkg/query-service/utils/logs_test.go index e1efd813d1..d40b9b8217 100644 --- a/pkg/query-service/utils/logs_test.go +++ b/pkg/query-service/utils/logs_test.go @@ -18,7 +18,9 @@ func TestLogsListTsRange(t *testing.T) { name: "testing for less then one hour", start: 1722262800000000000, // July 29, 2024 7:50:00 PM end: 1722263800000000000, // July 29, 2024 8:06:40 PM - res: []LogsListTsRange{}, + res: []LogsListTsRange{ + {1722262800000000000, 1722263800000000000}, + }, }, { name: "testing for more than one hour", From aeeb77bbc1f9fd042c40290d494baabc5abcea17 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 25 Oct 2024 16:13:44 +0530 Subject: [PATCH 11/41] feat: support for faster trace detail --- .../app/clickhouseReader/reader.go | 39 +++++++++++-------- pkg/query-service/model/response.go | 7 ++++ 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index fc5f3d03b1..3e1d0de16f 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -1725,22 +1725,32 @@ func (r *ClickHouseReader) GetUsage(ctx context.Context, queryParams *model.GetU } func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.SearchTracesParams) (*[]model.SearchSpansResult, error) { - var countSpans uint64 - // TODO(nitya): check if we can use timestamp filter here - countQuery := fmt.Sprintf("SELECT count() as count from %s.%s WHERE traceID=$1", r.TraceDB, r.traceIndexTableV3) - err := r.db.QueryRow(ctx, countQuery, params.TraceID).Scan(&countSpans) + searchSpansResult := []model.SearchSpansResult{ + { + Columns: []string{"__time", "SpanId", "TraceId", "ServiceName", "Name", "Kind", "DurationNano", "TagsKeys", "TagsValues", "References", "Events", "HasError", "StatusMessage", "StatusCodeString", "SpanKind"}, + IsSubTree: false, + Events: make([][]interface{}, 0), + }, + } + + var traceSummary model.TraceSummary + summaryQuery := fmt.Sprintf("SELECT * from %s.%s WHERE traceID=$1", r.TraceDB, "distributed_trace_summary") + err := r.db.QueryRow(ctx, summaryQuery, params.TraceID).Scan(&traceSummary.TraceID, &traceSummary.FirstReported, &traceSummary.LastReported, &traceSummary.NumSpans) if err != nil { + if err == sql.ErrNoRows { + return &searchSpansResult, nil + } zap.L().Error("Error in processing sql query", zap.Error(err)) return nil, fmt.Errorf("error in processing sql query") } - if countSpans > uint64(params.MaxSpansInTrace) { + if traceSummary.NumSpans > uint64(params.MaxSpansInTrace) { zap.L().Error("Max spans allowed in a trace limit reached", zap.Int("MaxSpansInTrace", params.MaxSpansInTrace), - zap.Uint64("Count", countSpans)) + zap.Uint64("Count", traceSummary.NumSpans)) userEmail, err := auth.GetEmailFromJwt(ctx) if err == nil { data := map[string]interface{}{ - "traceSize": countSpans, + "traceSize": traceSummary.NumSpans, "maxSpansInTraceLimit": params.MaxSpansInTrace, } telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_MAX_SPANS_ALLOWED_LIMIT_REACHED, data, userEmail, true, false) @@ -1751,7 +1761,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea userEmail, err := auth.GetEmailFromJwt(ctx) if err == nil { data := map[string]interface{}{ - "traceSize": countSpans, + "traceSize": traceSummary.NumSpans, } telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_TRACE_DETAIL_API, data, userEmail, true, false) } @@ -1759,11 +1769,11 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea var startTime, endTime, durationNano uint64 var searchScanResponses []model.SearchSpanResponseItemV2 - query := fmt.Sprintf("SELECT timestamp, durationNano, spanID, traceID, hasError, kind, serviceName, name, references, attributes_string, events, statusMessage, statusCodeString, spanKind FROM %s.%s WHERE traceID=$1", r.TraceDB, r.traceIndexTableV3) + query := fmt.Sprintf("SELECT timestamp, durationNano, spanID, traceID, hasError, kind, serviceName, name, references, attributes_string, events, statusMessage, statusCodeString, spanKind FROM %s.%s WHERE traceID=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceIndexTableV3) start := time.Now() - err = r.db.Select(ctx, &searchScanResponses, query, params.TraceID) + err = r.db.Select(ctx, &searchScanResponses, query, params.TraceID, strconv.FormatInt(traceSummary.FirstReported.Unix()-1800, 10), strconv.FormatInt(traceSummary.LastReported.Unix(), 10)) zap.L().Info(query) @@ -1773,13 +1783,8 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea } end := time.Now() zap.L().Debug("getTraceSQLQuery took: ", zap.Duration("duration", end.Sub(start))) - searchSpansResult := []model.SearchSpansResult{ - { - Columns: []string{"__time", "SpanId", "TraceId", "ServiceName", "Name", "Kind", "DurationNano", "TagsKeys", "TagsValues", "References", "Events", "HasError", "StatusMessage", "StatusCodeString", "SpanKind"}, - Events: make([][]interface{}, len(searchScanResponses)), - IsSubTree: false, - }, - } + + searchSpansResult[0].Events = make([][]interface{}, len(searchScanResponses)) searchSpanResponses := []model.SearchSpanResponseItem{} start = time.Now() diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 503770a257..5ac59a8bf5 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -286,6 +286,13 @@ type SearchSpanResponseItemV2 struct { SpanKind string `json:"spanKind" ch:"spanKind"` } +type TraceSummary struct { + TraceID string `json:"traceID" ch:"traceID"` + FirstReported time.Time `json:"firstReported" ch:"first_reported"` + LastReported time.Time `json:"lastReported" ch:"last_reported"` + NumSpans uint64 `json:"numSpans" ch:"num_spans"` +} + type OtelSpanRef struct { TraceId string `json:"traceId,omitempty"` SpanId string `json:"spanId,omitempty"` From e5d425f06ed64b6c787f38c44824fb2062a67771 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 25 Oct 2024 18:06:33 +0530 Subject: [PATCH 12/41] fix: searchTraces --- .../app/clickhouseReader/options.go | 3 ++ .../app/clickhouseReader/reader.go | 15 ++++++++-- pkg/query-service/model/response.go | 30 ++++++++++--------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index 8b75ab5cc9..72d54b3c0a 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -48,6 +48,7 @@ const ( defaultTraceIndexTableV3 string = "distributed_signoz_index_v3" defaultTraceResourceTableV3 string = "distributed_traces_v3_resource" + defaultTraceSummaryTable string = "distributed_trace_summary" ) // NamespaceConfig is Clickhouse's internal configuration data @@ -88,6 +89,7 @@ type namespaceConfig struct { TraceIndexTableV3 string TraceResourceTableV3 string + TraceSummaryTable string } // Connecto defines how to connect to the database @@ -183,6 +185,7 @@ func NewOptions( TraceIndexTableV3: defaultTraceIndexTableV3, TraceResourceTableV3: defaultTraceResourceTableV3, + TraceSummaryTable: defaultTraceSummaryTable, }, others: make(map[string]*namespaceConfig, len(otherNamespaces)), } diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 3e1d0de16f..eee4242f35 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -152,6 +152,7 @@ type ClickHouseReader struct { traceIndexTableV3 string traceResourceTableV3 string + traceSummaryTable string } // NewTraceReader returns a TraceReader for the database @@ -263,6 +264,7 @@ func NewReaderFromClickhouseConnection( traceIndexTableV3: options.primary.TraceIndexTableV3, traceResourceTableV3: options.primary.TraceResourceTableV3, + traceSummaryTable: options.primary.TraceSummaryTable, } } @@ -1734,7 +1736,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea } var traceSummary model.TraceSummary - summaryQuery := fmt.Sprintf("SELECT * from %s.%s WHERE traceID=$1", r.TraceDB, "distributed_trace_summary") + summaryQuery := fmt.Sprintf("SELECT * from %s.%s WHERE traceID=$1", r.TraceDB, r.traceSummaryTable) err := r.db.QueryRow(ctx, summaryQuery, params.TraceID).Scan(&traceSummary.TraceID, &traceSummary.FirstReported, &traceSummary.LastReported, &traceSummary.NumSpans) if err != nil { if err == sql.ErrNoRows { @@ -1796,7 +1798,14 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea return nil, err } - // TODO(nitya): add attributes_number and attributes_bool and resources_string to this + // merge attributes_number and attributes_bool to attributes_string + for k, v := range item.Attributes_bool { + item.Attributes_string[k] = fmt.Sprintf("%v", v) + } + for k, v := range item.Attributes_number { + item.Attributes_string[k] = fmt.Sprintf("%v", v) + } + jsonItem := model.SearchSpanResponseItem{ SpanID: item.SpanID, TraceID: item.TraceID, @@ -1810,7 +1819,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea SpanKind: item.SpanKind, References: ref, Events: item.Events, - TagMap: item.TagMap, + TagMap: item.Attributes_string, } jsonItem.TimeUnixNano = uint64(item.TimeUnixNano.UnixNano() / 1000000) diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 5ac59a8bf5..e0fbfc5860 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -270,20 +270,22 @@ type SearchSpanResponseItem struct { } type SearchSpanResponseItemV2 struct { - TimeUnixNano time.Time `json:"timestamp" ch:"timestamp"` - DurationNano uint64 `json:"durationNano" ch:"durationNano"` - SpanID string `json:"spanId" ch:"spanID"` - TraceID string `json:"traceId" ch:"traceID"` - HasError bool `json:"hasError" ch:"hasError"` - Kind int8 `json:"kind" ch:"kind"` - ServiceName string `json:"serviceName" ch:"serviceName"` - Name string `json:"name" ch:"name"` - References string `json:"references,omitempty" ch:"references"` - TagMap map[string]string `json:"tagMap" ch:"attributes_string"` - Events []string `json:"event" ch:"events"` - StatusMessage string `json:"statusMessage" ch:"statusMessage"` - StatusCodeString string `json:"statusCodeString" ch:"statusCodeString"` - SpanKind string `json:"spanKind" ch:"spanKind"` + TimeUnixNano time.Time `json:"timestamp" ch:"timestamp"` + DurationNano uint64 `json:"durationNano" ch:"durationNano"` + SpanID string `json:"spanId" ch:"spanID"` + TraceID string `json:"traceId" ch:"traceID"` + HasError bool `json:"hasError" ch:"hasError"` + Kind int8 `json:"kind" ch:"kind"` + ServiceName string `json:"serviceName" ch:"serviceName"` + Name string `json:"name" ch:"name"` + References string `json:"references,omitempty" ch:"references"` + Attributes_string map[string]string `json:"attributes_string" ch:"attributes_string"` + Attributes_number map[string]float64 `json:"attributes_number" ch:"attributes_number"` + Attributes_bool map[string]bool `json:"attributes_bool" ch:"attributes_bool"` + Events []string `json:"event" ch:"events"` + StatusMessage string `json:"statusMessage" ch:"statusMessage"` + StatusCodeString string `json:"statusCodeString" ch:"statusCodeString"` + SpanKind string `json:"spanKind" ch:"spanKind"` } type TraceSummary struct { From 25020edfb6f0d6337308e4e9a403abd3bf01771a Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 1 Nov 2024 15:04:48 +0530 Subject: [PATCH 13/41] fix: attribute enrichment updated and issue in group by --- pkg/query-service/app/traces/v4/enrich.go | 66 +++++++-- .../app/traces/v4/enrich_test.go | 140 +++++++++++------- .../app/traces/v4/query_builder.go | 10 +- .../app/traces/v4/query_builder_test.go | 10 +- 4 files changed, 157 insertions(+), 69 deletions(-) diff --git a/pkg/query-service/app/traces/v4/enrich.go b/pkg/query-service/app/traces/v4/enrich.go index 678913b659..dca10aaa7e 100644 --- a/pkg/query-service/app/traces/v4/enrich.go +++ b/pkg/query-service/app/traces/v4/enrich.go @@ -2,8 +2,55 @@ package v4 import v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +// we could have created aliases but as of now we are sticking to this as alias are not dynamic in nature +// Note: ALTER TABLE doesn't support adding an alias directly, so you may need to create the table with aliases from the start. +var attributeMatColsMapping = map[string]v3.AttributeKey{ + "http.route": { + Key: "httpRoute", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "messaging.system": { + Key: "msgSystem", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "messaging.peration": { + Key: "msgOperation", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "db.system": { + Key: "dbSystem", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "rpc.system": { + Key: "rpcSystem", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "rpc.service": { + Key: "rpcService", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "rpc.method": { + Key: "rpcMethod", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "peer.service": { + Key: "peerService", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, +} + func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) v3.AttributeKey { - // TODO(nitya) : update logic similar to logs + if matCol, ok := attributeMatColsMapping[key.Key]; ok { + return matCol + } if key.Type == "" || key.DataType == "" { // check if the key is present in the keys map if existingKey, ok := keys[key.Key]; ok { @@ -39,18 +86,13 @@ func EnrichTracesQuery(query *v3.BuilderQuery, keys map[string]v3.AttributeKey) query.Filters.Items[idx].Key = enrichKeyWithMetadata(filter.Key, keys) // if the serviceName column is used, use the corresponding resource attribute as well during filtering if filter.Key.Key == "serviceName" && filter.Key.IsColumn { - query.Filters.Items = append(query.Filters.Items, v3.FilterItem{ - Key: v3.AttributeKey{ - Key: "service.name", - DataType: v3.AttributeKeyDataTypeString, - Type: v3.AttributeKeyTypeResource, - IsColumn: false, - }, - Operator: filter.Operator, - Value: filter.Value, - }) + query.Filters.Items[idx].Key = v3.AttributeKey{ + Key: "service.name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeResource, + IsColumn: false, + } } - } } // enrich group by diff --git a/pkg/query-service/app/traces/v4/enrich_test.go b/pkg/query-service/app/traces/v4/enrich_test.go index 231e6c8a1e..4ee784e4c6 100644 --- a/pkg/query-service/app/traces/v4/enrich_test.go +++ b/pkg/query-service/app/traces/v4/enrich_test.go @@ -1,57 +1,97 @@ package v4 import ( + "reflect" + "testing" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" ) -var testColumnName = []struct { - Name string - AttributeKey v3.AttributeKey - ExpectedColumn string -}{ - { - Name: "resource", - AttributeKey: v3.AttributeKey{Key: "collector_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: false}, - ExpectedColumn: "resourceTagsMap['collector_id']", - }, - { - Name: "stringAttribute", - AttributeKey: v3.AttributeKey{Key: "customer_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, - ExpectedColumn: "stringTagMap['customer_id']", - }, - { - Name: "boolAttribute", - AttributeKey: v3.AttributeKey{Key: "has_error", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag, IsColumn: false}, - ExpectedColumn: "boolTagMap['has_error']", - }, - { - Name: "float64Attribute", - AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag, IsColumn: false}, - ExpectedColumn: "numberTagMap['count']", - }, - { - Name: "int64Attribute", - AttributeKey: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag, IsColumn: false}, - ExpectedColumn: "numberTagMap['count']", - }, - { - Name: "column", - AttributeKey: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, - ExpectedColumn: "name", - }, - { - Name: "missing key", - AttributeKey: v3.AttributeKey{Key: "xyz"}, - ExpectedColumn: "stringTagMap['xyz']", - }, +func TestEnrichTracesQuery(t *testing.T) { + type args struct { + query *v3.BuilderQuery + keys map[string]v3.AttributeKey + want *v3.BuilderQuery + } + tests := []struct { + name string + args args + }{ + { + name: "test 1", + args: args{ + query: &v3.BuilderQuery{ + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag}, Value: 100, Operator: ">"}, + }, + }, + }, + keys: map[string]v3.AttributeKey{ + "bytes": {Key: "bytes", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, + }, + want: &v3.BuilderQuery{ + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "bytes", Type: v3.AttributeKeyTypeTag, DataType: v3.AttributeKeyDataTypeInt64}, Value: 100, Operator: ">"}, + }, + }, + }, + }, + }, + { + name: "test service name", + args: args{ + query: &v3.BuilderQuery{ + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Value: "myservice", Operator: "="}, + }, + }, + }, + keys: map[string]v3.AttributeKey{}, + want: &v3.BuilderQuery{ + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "service.name", Type: v3.AttributeKeyTypeResource, DataType: v3.AttributeKeyDataTypeString}, Value: "myservice", Operator: "="}, + }, + }, + }, + }, + }, + { + name: "test mat attrs", + args: args{ + query: &v3.BuilderQuery{ + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "http.route", DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Value: "/api", Operator: "="}, + }, + }, + }, + keys: map[string]v3.AttributeKey{}, + want: &v3.BuilderQuery{ + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "httpRoute", DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Value: "/api", Operator: "="}, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + EnrichTracesQuery(tt.args.query, tt.args.keys) + if !reflect.DeepEqual(tt.args.query.Filters.Items[0].Key, tt.args.want.Filters.Items[0].Key) { + t.Errorf("EnrichTracesQuery() = %v, want %v", tt.args.query, tt.args.want) + } + }) + } } - -// func TestColumnName(t *testing.T) { -// for _, tt := range testColumnName { -// tt.AttributeKey = enrichKeyWithMetadata(tt.AttributeKey, map[string]v3.AttributeKey{}) -// Convey("testColumnName", t, func() { -// Column := getColumnName(tt.AttributeKey) -// So(Column, ShouldEqual, tt.ExpectedColumn) -// }) -// } -// } diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index aba2bc167f..ecfccb5b9b 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -235,6 +235,9 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. // timerange will be sent in epoch millisecond selectLabels := getSelectLabels(mq.GroupBy) + if selectLabels != "" { + selectLabels = selectLabels + "," + } orderBy := orderByAttributeKeyTags(panelType, mq.OrderBy, mq.GroupBy) if orderBy != "" { @@ -382,8 +385,11 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. // step is in seconds func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.QBOptions) (string, error) { // adjust the start and end time to the step interval - start = start - (start % (mq.StepInterval * 1000)) - end = end - (end % (mq.StepInterval * 1000)) + if panelType == v3.PanelTypeGraph { + // adjust the start and end time to the step interval for graph panel types + start = start - (start % (mq.StepInterval * 1000)) + end = end - (end % (mq.StepInterval * 1000)) + } if options.GraphLimitQtype == constants.FirstQueryGraphLimit { // give me just the group by names query, err := buildTracesQuery(start, end, mq.StepInterval, mq, panelType, options) diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index fa0f1a84ec..cbbb0da1e3 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -343,7 +343,7 @@ func Test_buildTracesQuery(t *testing.T) { {ColumnName: "host", Order: "ASC"}}, }, }, - want: "SELECT resources_number['host'] as `host` toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + want: "SELECT resources_number['host'] as `host`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_number['bytes'] > 100 AND " + "(resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND " + "(seen_at_ts_bucket_start <= 1680066458) AND simpleJSONExtractString(labels, 'service.name') = 'myService' AND labels like '%service.name%myService%' AND " + @@ -498,8 +498,8 @@ func TestPrepareTracesQuery(t *testing.T) { GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, - want: "SELECT `serviceName` from (SELECT `serviceName` as `serviceName` toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 " + - "where (timestamp >= '1680066360726180000' AND timestamp <= '1680066457999980000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066457) group by `serviceName`) LIMIT 10", + want: "SELECT `serviceName` from (SELECT `serviceName` as `serviceName`, toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) group by `serviceName`) LIMIT 10", }, { name: "test with limit - second", @@ -519,8 +519,8 @@ func TestPrepareTracesQuery(t *testing.T) { GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, - want: "SELECT `serviceName` as `serviceName` toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 where " + - "(timestamp >= '1680066360726180000' AND timestamp <= '1680066457999980000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066457) AND (`serviceName`) GLOBAL IN (%s) group by `serviceName`", + want: "SELECT `serviceName` as `serviceName`, toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 where " + + "(timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND (`serviceName`) GLOBAL IN (%s) group by `serviceName`", }, } From 753eb0847ecefdb27917f792715d1aa72cc4b59a Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 1 Nov 2024 17:31:23 +0530 Subject: [PATCH 14/41] fix: issues in group by --- .../app/traces/v4/query_builder.go | 12 +++++------ .../app/traces/v4/query_builder_test.go | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index ecfccb5b9b..c7c8cfb75f 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -219,6 +219,12 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. return "", err } + emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(mq.GroupBy) + if err != nil { + return "", err + } + filterSubQuery += emptyValuesInGroupByFilter + if filterSubQuery != "" { filterSubQuery = " AND " + filterSubQuery } @@ -310,12 +316,6 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. queryTmpl = "SELECT " + tracesV3.GetSelectKeys(mq.AggregateOperator, mq.GroupBy) + " from (" + queryTmpl + ")" } - emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(mq.GroupBy) - if err != nil { - return "", err - } - filterSubQuery += emptyValuesInGroupByFilter - if options.GraphLimitQtype == constants.SecondQueryGraphLimit { filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", tracesV3.GetSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "%s)" } diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index cbbb0da1e3..3598a2864c 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -323,6 +323,27 @@ func Test_buildTracesQuery(t *testing.T) { want string wantErr bool }{ + { + name: "Test buildTracesQuery", + args: args{ + panelType: v3.PanelTypeTable, + start: 1680066360726210000, + end: 1680066458000000000, + step: 1000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorCount, + Filters: &v3.FilterSet{ + Items: []v3.FilterItem{}, + }, + GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{ + {ColumnName: "http.method", Order: "ASC"}}, + }, + }, + want: "SELECT attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "AND mapContains(attributes_string, 'http.method') group by `http.method` order by `http.method` ASC", + }, { name: "Test buildTracesQuery", args: args{ From f4103550884eeb508b3a9b4b10c9da0492159167 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 5 Nov 2024 21:44:39 +0530 Subject: [PATCH 15/41] fix: enrichment using alias --- .../app/clickhouseReader/options.go | 18 +- .../app/clickhouseReader/reader.go | 157 ++++++------------ pkg/query-service/app/logs/v3/enrich_query.go | 2 +- pkg/query-service/app/traces/v4/enrich.go | 92 ++++------ .../app/traces/v4/query_builder.go | 27 ++- pkg/query-service/constants/constants.go | 143 ++++++++++++++++ pkg/query-service/contextlinks/links.go | 2 +- pkg/query-service/utils/logs.go | 4 +- 8 files changed, 266 insertions(+), 179 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/options.go b/pkg/query-service/app/clickhouseReader/options.go index 72d54b3c0a..25eea0c7ff 100644 --- a/pkg/query-service/app/clickhouseReader/options.go +++ b/pkg/query-service/app/clickhouseReader/options.go @@ -22,6 +22,7 @@ const ( defaultTraceDB string = "signoz_traces" defaultOperationsTable string = "distributed_signoz_operations" defaultIndexTable string = "distributed_signoz_index_v2" + defaultLocalIndexTable string = "signoz_index_v2" defaultErrorTable string = "distributed_signoz_error_index_v2" defaultDurationTable string = "distributed_durationSort" defaultUsageExplorerTable string = "distributed_usage_explorer" @@ -47,6 +48,7 @@ const ( defaultLogsResourceTableV2 string = "distributed_logs_v2_resource" defaultTraceIndexTableV3 string = "distributed_signoz_index_v3" + defaultTraceLocalTableName string = "signoz_index_v3" defaultTraceResourceTableV3 string = "distributed_traces_v3_resource" defaultTraceSummaryTable string = "distributed_trace_summary" ) @@ -62,6 +64,7 @@ type namespaceConfig struct { TraceDB string OperationsTable string IndexTable string + LocalIndexTable string DurationTable string UsageExplorerTable string SpansTable string @@ -87,9 +90,10 @@ type namespaceConfig struct { LogsResourceLocalTableV2 string LogsResourceTableV2 string - TraceIndexTableV3 string - TraceResourceTableV3 string - TraceSummaryTable string + TraceIndexTableV3 string + TraceLocalTableNameV3 string + TraceResourceTableV3 string + TraceSummaryTable string } // Connecto defines how to connect to the database @@ -158,6 +162,7 @@ func NewOptions( TraceDB: defaultTraceDB, OperationsTable: defaultOperationsTable, IndexTable: defaultIndexTable, + LocalIndexTable: defaultLocalIndexTable, ErrorTable: defaultErrorTable, DurationTable: defaultDurationTable, UsageExplorerTable: defaultUsageExplorerTable, @@ -183,9 +188,10 @@ func NewOptions( LogsResourceTableV2: defaultLogsResourceTableV2, LogsResourceLocalTableV2: defaultLogsResourceLocalTableV2, - TraceIndexTableV3: defaultTraceIndexTableV3, - TraceResourceTableV3: defaultTraceResourceTableV3, - TraceSummaryTable: defaultTraceSummaryTable, + TraceIndexTableV3: defaultTraceIndexTableV3, + TraceLocalTableNameV3: defaultTraceLocalTableName, + TraceResourceTableV3: defaultTraceResourceTableV3, + TraceSummaryTable: defaultTraceSummaryTable, }, others: make(map[string]*namespaceConfig, len(otherNamespaces)), } diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 2b2a29527e..0b9a41d549 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -145,12 +145,14 @@ type ClickHouseReader struct { liveTailRefreshSeconds int cluster string - useLogsNewSchema bool - useTraceNewSchema bool + useLogsNewSchema bool + useTraceNewSchema bool + logsTableName string logsLocalTableName string - traceIndexTableV3 string + traceTableName string + traceLocalTableName string traceResourceTableV3 string traceSummaryTable string } @@ -225,6 +227,13 @@ func NewReaderFromClickhouseConnection( logsLocalTableName = options.primary.LogsLocalTableV2 } + traceTableName := options.primary.IndexTable + traceLocalTableName := options.primary.LocalIndexTable + if useTraceNewSchema { + traceTableName = options.primary.TraceIndexTableV3 + traceLocalTableName = options.primary.TraceLocalTableNameV3 + } + return &ClickHouseReader{ db: wrap, localDB: localDB, @@ -262,7 +271,8 @@ func NewReaderFromClickhouseConnection( logsTableName: logsTableName, logsLocalTableName: logsLocalTableName, - traceIndexTableV3: options.primary.TraceIndexTableV3, + traceLocalTableName: traceLocalTableName, + traceTableName: traceTableName, traceResourceTableV3: options.primary.TraceResourceTableV3, traceSummaryTable: options.primary.TraceSummaryTable, } @@ -475,13 +485,8 @@ func (r *ClickHouseReader) GetQueryRangeResult(ctx context.Context, query *model } func (r *ClickHouseReader) GetServicesList(ctx context.Context) (*[]string, error) { - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } - services := []string{} - query := fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE ts_bucket_start > (toUnixTimestamp(now()) - 1800) AND toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, indexTable) + query := fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE ts_bucket_start > (toUnixTimestamp(now()) - 1800) AND toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceLocalTableName) rows, err := r.db.Query(ctx, query) @@ -543,10 +548,6 @@ func (r *ClickHouseReader) GetTopLevelOperations(ctx context.Context, skipConfig } func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.GetServicesParams, skipConfig *model.SkipConfig) (*[]model.ServiceItem, *model.ApiError) { - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } if r.indexTable == "" { return nil, &model.ApiError{Typ: model.ErrorExec, Err: ErrNoIndexTable} @@ -594,14 +595,14 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G count(*) as numCalls FROM %s.%s WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end`, - r.TraceDB, indexTable, + r.TraceDB, r.traceTableName, ) errorQuery := fmt.Sprintf( `SELECT count(*) as numErrors FROM %s.%s WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end AND statusCode=2`, - r.TraceDB, indexTable, + r.TraceDB, r.traceTableName, ) args := []interface{}{} @@ -676,11 +677,6 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams *model.GetServiceOverviewParams, skipConfig *model.SkipConfig) (*[]model.ServiceOverviewItem, *model.ApiError) { - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } - topLevelOps, apiErr := r.GetTopLevelOperations(ctx, skipConfig, *queryParams.Start, *queryParams.End, nil) if apiErr != nil { return nil, apiErr @@ -709,7 +705,7 @@ func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams * count(*) as numCalls FROM %s.%s WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end`, - r.TraceDB, indexTable, + r.TraceDB, r.traceTableName, ) args := []interface{}{} args = append(args, namedArgs...) @@ -740,7 +736,7 @@ func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams * count(*) as numErrors FROM %s.%s WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end AND statusCode=2`, - r.TraceDB, indexTable, + r.TraceDB, r.traceTableName, ) args = []interface{}{} args = append(args, namedArgs...) @@ -804,11 +800,6 @@ func buildFilterArrayQuery(_ context.Context, excludeMap map[string]struct{}, pa func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *model.SpanFilterParams) (*model.SpanFiltersResponse, *model.ApiError) { - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } - var query string excludeMap := make(map[string]struct{}) for _, e := range queryParams.Exclude { @@ -882,7 +873,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode case constants.TraceID: continue case constants.ServiceName: - finalQuery := fmt.Sprintf("SELECT serviceName, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) + finalQuery := fmt.Sprintf("SELECT serviceName, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) finalQuery += query finalQuery += " GROUP BY serviceName" var dBResponse []model.DBResponseServiceName @@ -899,7 +890,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.HttpRoute: - finalQuery := fmt.Sprintf("SELECT httpRoute, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) + finalQuery := fmt.Sprintf("SELECT httpRoute, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) finalQuery += query finalQuery += " GROUP BY httpRoute" var dBResponse []model.DBResponseHttpRoute @@ -916,7 +907,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.HttpUrl: - finalQuery := fmt.Sprintf("SELECT httpUrl, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) + finalQuery := fmt.Sprintf("SELECT httpUrl, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) finalQuery += query finalQuery += " GROUP BY httpUrl" var dBResponse []model.DBResponseHttpUrl @@ -933,7 +924,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.HttpMethod: - finalQuery := fmt.Sprintf("SELECT httpMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) + finalQuery := fmt.Sprintf("SELECT httpMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) finalQuery += query finalQuery += " GROUP BY httpMethod" var dBResponse []model.DBResponseHttpMethod @@ -950,7 +941,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.HttpHost: - finalQuery := fmt.Sprintf("SELECT httpHost, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) + finalQuery := fmt.Sprintf("SELECT httpHost, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) finalQuery += query finalQuery += " GROUP BY httpHost" var dBResponse []model.DBResponseHttpHost @@ -967,7 +958,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.OperationRequest: - finalQuery := fmt.Sprintf("SELECT name, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) + finalQuery := fmt.Sprintf("SELECT name, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) finalQuery += query finalQuery += " GROUP BY name" var dBResponse []model.DBResponseOperation @@ -984,7 +975,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.Status: - finalQuery := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = true", r.TraceDB, indexTable) + finalQuery := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = true", r.TraceDB, r.traceTableName) finalQuery += query var dBResponse []model.DBResponseTotal err := r.db.Select(ctx, &dBResponse, finalQuery, args...) @@ -995,7 +986,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} } - finalQuery2 := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = false", r.TraceDB, indexTable) + finalQuery2 := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = false", r.TraceDB, r.traceTableName) finalQuery2 += query var dBResponse2 []model.DBResponseTotal err = r.db.Select(ctx, &dBResponse2, finalQuery2, args...) @@ -1020,7 +1011,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode finalQuery := "" if !durationSortEnabled { // if duration sort is not enabled, we need to get the min and max duration from the index table - finalQuery = fmt.Sprintf("SELECT min(durationNano) as min, max(durationNano) as max FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) + finalQuery = fmt.Sprintf("SELECT min(durationNano) as min, max(durationNano) as max FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) finalQuery += query var dBResponse []model.DBResponseMinMax err = r.db.Select(ctx, &dBResponse, finalQuery, args...) @@ -1065,7 +1056,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } } case constants.RPCMethod: - finalQuery := fmt.Sprintf("SELECT rpcMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) + finalQuery := fmt.Sprintf("SELECT rpcMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) finalQuery += query finalQuery += " GROUP BY rpcMethod" var dBResponse []model.DBResponseRPCMethod @@ -1083,7 +1074,7 @@ func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *mode } case constants.ResponseStatusCode: - finalQuery := fmt.Sprintf("SELECT responseStatusCode, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, indexTable) + finalQuery := fmt.Sprintf("SELECT responseStatusCode, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) finalQuery += query finalQuery += " GROUP BY responseStatusCode" var dBResponse []model.DBResponseStatusCodeMethod @@ -1131,12 +1122,7 @@ func getStatusFilters(query string, statusParams []string, excludeMap map[string func (r *ClickHouseReader) GetFilteredSpans(ctx context.Context, queryParams *model.GetFilteredSpansParams) (*model.GetFilterSpansResponse, *model.ApiError) { - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } - - queryTable := fmt.Sprintf("%s.%s", r.TraceDB, indexTable) + queryTable := fmt.Sprintf("%s.%s", r.TraceDB, r.traceTableName) excludeMap := make(map[string]struct{}) for _, e := range queryParams.Exclude { @@ -1427,11 +1413,6 @@ func addExistsOperator(item model.TagQuery, tagMapType string, not bool) (string func (r *ClickHouseReader) GetTagFilters(ctx context.Context, queryParams *model.TagFilterParams) (*model.TagFilters, *model.ApiError) { - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } - excludeMap := make(map[string]struct{}) for _, e := range queryParams.Exclude { if e == constants.OperationRequest { @@ -1488,7 +1469,7 @@ func (r *ClickHouseReader) GetTagFilters(ctx context.Context, queryParams *model tagFilters := []model.TagFilters{} // Alternative finalQuery := fmt.Sprintf(`SELECT DISTINCT arrayJoin(tagMap.keys) as tagKeys FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, indexTable) - finalQuery := fmt.Sprintf(`SELECT groupUniqArrayArray(mapKeys(stringTagMap)) as stringTagKeys, groupUniqArrayArray(mapKeys(numberTagMap)) as numberTagKeys, groupUniqArrayArray(mapKeys(boolTagMap)) as boolTagKeys FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, indexTable) + finalQuery := fmt.Sprintf(`SELECT groupUniqArrayArray(mapKeys(stringTagMap)) as stringTagKeys, groupUniqArrayArray(mapKeys(numberTagMap)) as numberTagKeys, groupUniqArrayArray(mapKeys(boolTagMap)) as boolTagKeys FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, r.traceTableName) finalQuery += query err := r.db.Select(ctx, &tagFilters, finalQuery, args...) @@ -1536,11 +1517,6 @@ func excludeTags(_ context.Context, tags []string) []string { func (r *ClickHouseReader) GetTagValues(ctx context.Context, queryParams *model.TagFilterParams) (*model.TagValues, *model.ApiError) { - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } - if queryParams.TagKey.Type == model.TagTypeNumber { return &model.TagValues{ NumberTagValues: make([]float64, 0), @@ -1604,7 +1580,7 @@ func (r *ClickHouseReader) GetTagValues(ctx context.Context, queryParams *model. tagValues := []model.TagValues{} - finalQuery := fmt.Sprintf(`SELECT groupArray(DISTINCT attributes_string[@key]) as stringTagValues FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, indexTable) + finalQuery := fmt.Sprintf(`SELECT groupArray(DISTINCT attributes_string[@key]) as stringTagValues FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, r.traceTableName) finalQuery += query finalQuery += " LIMIT @limit" @@ -1636,10 +1612,6 @@ func (r *ClickHouseReader) GetTagValues(ctx context.Context, queryParams *model. } func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *model.GetTopOperationsParams) (*[]model.TopOperationsItem, *model.ApiError) { - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } namedArgs := []interface{}{ clickhouse.Named("start", strconv.FormatInt(queryParams.Start.UnixNano(), 10)), @@ -1659,7 +1631,7 @@ func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *mo name FROM %s.%s WHERE serviceName = @serviceName AND timestamp>= @start AND timestamp<= @end`, - r.TraceDB, indexTable, + r.TraceDB, r.traceTableName, ) args := []interface{}{} args = append(args, namedArgs...) @@ -1771,7 +1743,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea var startTime, endTime, durationNano uint64 var searchScanResponses []model.SearchSpanResponseItemV2 - query := fmt.Sprintf("SELECT timestamp, durationNano, spanID, traceID, hasError, kind, serviceName, name, references, attributes_string, events, statusMessage, statusCodeString, spanKind FROM %s.%s WHERE traceID=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceIndexTableV3) + query := fmt.Sprintf("SELECT timestamp, durationNano, spanID, traceID, hasError, kind, serviceName, name, references, attributes_string, events, statusMessage, statusCodeString, spanKind FROM %s.%s WHERE traceID=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceTableName) start := time.Now() @@ -2012,12 +1984,6 @@ func (r *ClickHouseReader) GetDependencyGraph(ctx context.Context, queryParams * } func (r *ClickHouseReader) GetFilteredSpansAggregates(ctx context.Context, queryParams *model.GetFilteredSpanAggregatesParams) (*model.GetFilteredSpansAggregatesResponse, *model.ApiError) { - - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } - excludeMap := make(map[string]struct{}) for _, e := range queryParams.Exclude { if e == constants.OperationRequest { @@ -2063,7 +2029,7 @@ func (r *ClickHouseReader) GetFilteredSpansAggregates(ctx context.Context, query // Using %s for groupBy params as it can be a custom column and custom columns are not supported by clickhouse-go yet: // issue link: https://github.com/ClickHouse/clickhouse-go/issues/870 if queryParams.GroupBy != "" && columnExists { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, queryParams.GroupBy, aggregation_query, r.TraceDB, indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, queryParams.GroupBy, aggregation_query, r.TraceDB, r.traceTableName) args = append(args, clickhouse.Named("groupByVar", queryParams.GroupBy)) } else if queryParams.GroupBy != "" { customStr = strings.Split(queryParams.GroupBy, ".(") @@ -2071,17 +2037,17 @@ func (r *ClickHouseReader) GetFilteredSpansAggregates(ctx context.Context, query return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("GroupBy: %s not supported", queryParams.GroupBy)} } if customStr[1] == string(model.TagTypeString)+")" { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, stringTagMap['%s'] as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, stringTagMap['%s'] as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, r.traceTableName) } else if customStr[1] == string(model.TagTypeNumber)+")" { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(numberTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(numberTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, r.traceTableName) } else if customStr[1] == string(model.TagTypeBool)+")" { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(boolTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(boolTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, r.traceTableName) } else { // return error for unsupported group by return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("GroupBy: %s not supported", queryParams.GroupBy)} } } else { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, aggregation_query, r.TraceDB, indexTable) + query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, aggregation_query, r.TraceDB, r.traceTableName) } if len(queryParams.TraceID) > 0 { @@ -3249,15 +3215,10 @@ func (r *ClickHouseReader) GetLogsInfoInLastHeartBeatInterval(ctx context.Contex } func (r *ClickHouseReader) GetTagsInfoInLastHeartBeatInterval(ctx context.Context, interval time.Duration) (*model.TagsInfo, error) { - indexTable := r.indexTable - if r.useTraceNewSchema { - indexTable = r.traceIndexTableV3 - } - queryStr := fmt.Sprintf(`select serviceName, stringTagMap['deployment.environment'] as env, stringTagMap['telemetry.sdk.language'] as language from %s.%s where timestamp > toUnixTimestamp(now()-toIntervalMinute(%d)) - group by serviceName, env, language;`, r.TraceDB, indexTable, int(interval.Minutes())) + group by serviceName, env, language;`, r.TraceDB, r.traceTableName, int(interval.Minutes())) tagTelemetryDataList := []model.TagTelemetryData{} err := r.db.Select(ctx, &tagTelemetryDataList, queryStr) @@ -4772,8 +4733,6 @@ func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req if err := rows.Scan(&tagKey, &tagType, &dataType, &isColumn); err != nil { return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) } - // TODO: Remove this once the column name are updated in the table - tagKey = tempHandleFixedColumns(tagKey) key := v3.AttributeKey{ Key: tagKey, DataType: v3.AttributeKeyDataType(dataType), @@ -4813,8 +4772,6 @@ func (r *ClickHouseReader) GetTraceAttributeKeys(ctx context.Context, req *v3.Fi if err := rows.Scan(&tagKey, &tagType, &dataType, &isColumn); err != nil { return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) } - // TODO: Remove this once the column name are updated in the table - tagKey = tempHandleFixedColumns(tagKey) key := v3.AttributeKey{ Key: tagKey, DataType: v3.AttributeKeyDataType(dataType), @@ -4826,19 +4783,6 @@ func (r *ClickHouseReader) GetTraceAttributeKeys(ctx context.Context, req *v3.Fi return &response, nil } -// tempHandleFixedColumns is a temporary function to handle the fixed columns whose name has been changed in AttributeKeys Table -func tempHandleFixedColumns(tagKey string) string { - switch { - case tagKey == "traceId": - tagKey = "traceID" - case tagKey == "spanId": - tagKey = "spanID" - case tagKey == "parentSpanId": - tagKey = "parentSpanID" - } - return tagKey -} - func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { var query string @@ -4899,31 +4843,38 @@ func (r *ClickHouseReader) GetSpanAttributeKeys(ctx context.Context) (map[string var rows driver.Rows response := map[string]v3.AttributeKey{} - query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, dataType, isColumn FROM %s.%s", r.TraceDB, r.spanAttributesKeysTable) + query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, dataType FROM %s.%s", r.TraceDB, r.spanAttributesKeysTable) rows, err = r.db.Query(ctx, query) - if err != nil { zap.L().Error("Error while executing query", zap.Error(err)) return nil, fmt.Errorf("error while executing query: %s", err.Error()) } defer rows.Close() + statements := []model.ShowCreateTableStatement{} + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.TraceDB, r.traceTableName) + err = r.db.Select(ctx, &statements, query) + if err != nil { + return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) + } + var tagKey string var dataType string var tagType string - var isColumn bool for rows.Next() { - if err := rows.Scan(&tagKey, &tagType, &dataType, &isColumn); err != nil { + if err := rows.Scan(&tagKey, &tagType, &dataType); err != nil { return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) } key := v3.AttributeKey{ Key: tagKey, DataType: v3.AttributeKeyDataType(dataType), Type: v3.AttributeKeyType(tagType), - IsColumn: isColumn, + IsColumn: isColumn(r.useLogsNewSchema, statements[0].Statement, tagType, tagKey, dataType), } - response[tagKey] = key + + name := tagKey + "##" + tagType + "##" + strings.ToLower(dataType) + response[name] = key } return response, nil } diff --git a/pkg/query-service/app/logs/v3/enrich_query.go b/pkg/query-service/app/logs/v3/enrich_query.go index 49ff422fd5..57fced3db2 100644 --- a/pkg/query-service/app/logs/v3/enrich_query.go +++ b/pkg/query-service/app/logs/v3/enrich_query.go @@ -142,7 +142,7 @@ func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.Attribu } // check if the field is present in the fields map - for _, key := range utils.GenerateLogEnrichmentKeys(field) { + for _, key := range utils.GenerateEnrichmentKeys(field) { if val, ok := fields[key]; ok { return val } diff --git a/pkg/query-service/app/traces/v4/enrich.go b/pkg/query-service/app/traces/v4/enrich.go index dca10aaa7e..69dbb2e49c 100644 --- a/pkg/query-service/app/traces/v4/enrich.go +++ b/pkg/query-service/app/traces/v4/enrich.go @@ -1,69 +1,47 @@ package v4 -import v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +import ( + "go.signoz.io/signoz/pkg/query-service/constants" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" +) -// we could have created aliases but as of now we are sticking to this as alias are not dynamic in nature -// Note: ALTER TABLE doesn't support adding an alias directly, so you may need to create the table with aliases from the start. -var attributeMatColsMapping = map[string]v3.AttributeKey{ - "http.route": { - Key: "httpRoute", - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, - "messaging.system": { - Key: "msgSystem", - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, - "messaging.peration": { - Key: "msgOperation", - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, - "db.system": { - Key: "dbSystem", - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, - "rpc.system": { - Key: "rpcSystem", - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, - "rpc.service": { - Key: "rpcService", - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, - "rpc.method": { - Key: "rpcMethod", - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, - "peer.service": { - Key: "peerService", - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, +func isEnriched(field v3.AttributeKey) bool { + // if it is timestamp/id dont check + if field.Key == "timestamp" || field.Key == "id" || field.Key == constants.SigNozOrderByValue { + return true + } + + // don't need to enrich the static fields as they will be always used a column + if _, ok := constants.StaticFieldsTraces[field.Key]; ok && field.IsColumn { + return true + } + + return false } func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey) v3.AttributeKey { - if matCol, ok := attributeMatColsMapping[key.Key]; ok { - return matCol + if isEnriched(key) { + return key + } + + if v, ok := constants.StaticFieldsTraces[key.Key]; ok { + return v } - if key.Type == "" || key.DataType == "" { - // check if the key is present in the keys map - if existingKey, ok := keys[key.Key]; ok { - key.IsColumn = existingKey.IsColumn - key.Type = existingKey.Type - key.DataType = existingKey.DataType - } else { // if not present then set the default values - key.Type = v3.AttributeKeyTypeTag - key.DataType = v3.AttributeKeyDataTypeString - key.IsColumn = false - return key + + for _, key := range utils.GenerateEnrichmentKeys(key) { + if val, ok := keys[key]; ok { + return val } } + + // enrich with default values if metadata is not found + if key.Type == "" { + key.Type = v3.AttributeKeyTypeTag + } + if key.DataType == "" { + key.DataType = v3.AttributeKeyDataTypeString + } return key } diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index c7c8cfb75f..201f2bf752 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -53,15 +53,24 @@ func getClickHouseTracesColumnDataType(columnDataType v3.AttributeKeyDataType) s } func getColumnName(key v3.AttributeKey) string { - // TODO (nitya): - // consider routing things like serviceName col to service.name resource attribute for filtering - // consider using static details for some columns - if key.IsColumn { - return "`" + key.Key + "`" - } - keyType := getClickHouseTracesColumnType(key.Type) - keyDType := getClickHouseTracesColumnDataType(key.DataType) - return fmt.Sprintf("%s_%s['%s']", keyType, keyDType, key.Key) + if !key.IsColumn { + keyType := getClickHouseTracesColumnType(key.Type) + keyDType := getClickHouseTracesColumnDataType(key.DataType) + return fmt.Sprintf("%s_%s['%s']", keyType, keyDType, key.Key) + } + + // check if it is a static field + if key.Type == v3.AttributeKeyTypeUnspecified { + // name is the column name + return key.Key + } + + // if key present in static return as it is + if _, ok := constants.StaticFieldsTraces[key.Key]; ok { + return key.Key + } + + return utils.GetClickhouseColumnName(string(key.Type), string(key.DataType), key.Key) } // getSelectLabels returns the select labels for the query based on groupBy and aggregateOperator diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 4b91a6e5ea..1f0371e4a7 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -432,3 +432,146 @@ const MaxFilterSuggestionsExamplesLimit = 10 var SpanRenderLimitStr = GetOrDefaultEnv("SPAN_RENDER_LIMIT", "2500") var MaxSpansInTraceStr = GetOrDefaultEnv("MAX_SPANS_IN_TRACE", "250000") + +var StaticFieldsTraces = map[string]v3.AttributeKey{ + "traceID": { + Key: "traceID", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "spanID": { + Key: "spanID", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "parentSpanID": { + Key: "parentSpanID", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "name": { + Key: "name", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "serviceName": { + Key: "serviceName", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "kind": { + Key: "kind", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "spanKind": { + Key: "spanKind", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "durationNano": { + Key: "durationNano", + DataType: v3.AttributeKeyDataTypeFloat64, + IsColumn: true, + }, + "statusCode": { + Key: "statusCode", + DataType: v3.AttributeKeyDataTypeFloat64, + IsColumn: true, + }, + "hasError": { + Key: "hasError", + DataType: v3.AttributeKeyDataTypeBool, + IsColumn: true, + }, + "statusMessage": { + Key: "statusMessage", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "statusCodeString": { + Key: "statusCodeString", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "externalHttpMethod": { + Key: "externalHttpMethod", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "externalHttpUrl": { + Key: "externalHttpUrl", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "dbSystem": { + Key: "dbSystem", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "dbName": { + Key: "dbName", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "dbOperation": { + Key: "dbOperation", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "peerService": { + Key: "peerService", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "httpMethod": { + Key: "httpMethod", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "httpUrl": { + Key: "httpUrl", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "httpRoute": { + Key: "httpRoute", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "httpHost": { + Key: "httpHost", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "msgSystem": { + Key: "msgSystem", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "msgOperation": { + Key: "msgOperation", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "rpcSystem": { + Key: "rpcSystem", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "rpcService": { + Key: "rpcService", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "rpcMethod": { + Key: "rpcMethod", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "responseStatusCode": { + Key: "responseStatusCode", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, +} diff --git a/pkg/query-service/contextlinks/links.go b/pkg/query-service/contextlinks/links.go index 260745eda3..9e48dfb1a2 100644 --- a/pkg/query-service/contextlinks/links.go +++ b/pkg/query-service/contextlinks/links.go @@ -183,7 +183,7 @@ func PrepareFilters(labels map[string]string, whereClauseItems []v3.FilterItem, var attrFound bool // as of now this logic will only apply for logs - for _, tKey := range utils.GenerateLogEnrichmentKeys(v3.AttributeKey{Key: key}) { + for _, tKey := range utils.GenerateEnrichmentKeys(v3.AttributeKey{Key: key}) { if val, ok := keys[tKey]; ok { attributeKey = val attrFound = true diff --git a/pkg/query-service/utils/logs.go b/pkg/query-service/utils/logs.go index b7bf8d9491..11129ae3d8 100644 --- a/pkg/query-service/utils/logs.go +++ b/pkg/query-service/utils/logs.go @@ -42,8 +42,8 @@ func GetLogsListTsRanges(start, end int64) []LogsListTsRange { } // This tries to see all possible fields that it can fall back to if some meta is missing -// check Test_GenerateLogEnrichmentKeys for example -func GenerateLogEnrichmentKeys(field v3.AttributeKey) []string { +// check Test_GenerateEnrichmentKeys for example +func GenerateEnrichmentKeys(field v3.AttributeKey) []string { names := []string{} if field.Type != v3.AttributeKeyTypeUnspecified && field.DataType != v3.AttributeKeyDataTypeUnspecified { names = append(names, field.Key+"##"+field.Type.String()+"##"+field.DataType.String()) From aee3ca4fb11de8df08462604107a585962cd5a1d Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 6 Nov 2024 09:53:47 +0530 Subject: [PATCH 16/41] fix: test file added --- pkg/query-service/utils/logs_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/utils/logs_test.go b/pkg/query-service/utils/logs_test.go index d40b9b8217..4a69534f80 100644 --- a/pkg/query-service/utils/logs_test.go +++ b/pkg/query-service/utils/logs_test.go @@ -55,7 +55,7 @@ func TestLogsListTsRange(t *testing.T) { } } -func Test_GenerateLogEnrichmentKeys(t *testing.T) { +func Test_GenerateEnrichmentKeys(t *testing.T) { type args struct { field v3.AttributeKey } @@ -98,8 +98,8 @@ func Test_GenerateLogEnrichmentKeys(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := GenerateLogEnrichmentKeys(tt.args.field); !reflect.DeepEqual(got, tt.want) { - t.Errorf("generateLogEnrichmentKeys() = %v, want %v", got, tt.want) + if got := GenerateEnrichmentKeys(tt.args.field); !reflect.DeepEqual(got, tt.want) { + t.Errorf("generateEnrichmentKeys() = %v, want %v", got, tt.want) } }) } From 0418bfff0e9245ca38ed7ef30bf28c5b118f55a2 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 8 Nov 2024 13:01:01 +0530 Subject: [PATCH 17/41] fix: tests --- .../app/traces/v4/enrich_test.go | 4 +- .../app/traces/v4/query_builder.go | 2 +- .../app/traces/v4/query_builder_test.go | 40 ++++--------------- .../rules/threshold_rule_test.go | 2 +- 4 files changed, 11 insertions(+), 37 deletions(-) diff --git a/pkg/query-service/app/traces/v4/enrich_test.go b/pkg/query-service/app/traces/v4/enrich_test.go index 4ee784e4c6..063fa5c72d 100644 --- a/pkg/query-service/app/traces/v4/enrich_test.go +++ b/pkg/query-service/app/traces/v4/enrich_test.go @@ -29,7 +29,7 @@ func TestEnrichTracesQuery(t *testing.T) { }, }, keys: map[string]v3.AttributeKey{ - "bytes": {Key: "bytes", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, + "bytes##tag##int64": {Key: "bytes", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, }, want: &v3.BuilderQuery{ Filters: &v3.FilterSet{ @@ -79,7 +79,7 @@ func TestEnrichTracesQuery(t *testing.T) { Filters: &v3.FilterSet{ Operator: "AND", Items: []v3.FilterItem{ - {Key: v3.AttributeKey{Key: "httpRoute", DataType: v3.AttributeKeyDataTypeString, IsColumn: true}, Value: "/api", Operator: "="}, + {Key: v3.AttributeKey{Key: "http.route", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "/api", Operator: "="}, }, }, }, diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index 201f2bf752..e7fbf83f4b 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -70,7 +70,7 @@ func getColumnName(key v3.AttributeKey) string { return key.Key } - return utils.GetClickhouseColumnName(string(key.Type), string(key.DataType), key.Key) + return "`" + utils.GetClickhouseColumnNameV2(string(key.Type), string(key.DataType), key.Key) + "`" } // getSelectLabels returns the select labels for the query based on groupBy and aggregateOperator diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index 3598a2864c..6094ac9892 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -108,7 +108,7 @@ func Test_getColumnName(t *testing.T) { args: args{ key: v3.AttributeKey{Key: "data", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, }, - want: "`data`", + want: "`attribute_string_data`", }, { name: "missing meta", @@ -151,7 +151,7 @@ func Test_getSelectLabels(t *testing.T) { {Key: "service_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: true}, }, }, - want: " attributes_string['user_name'] as `user_name`, `service_name` as `service_name`", + want: " attributes_string['user_name'] as `user_name`, `resource_string_service_name` as `service_name`", }, } for _, tt := range tests { @@ -214,7 +214,7 @@ func Test_buildTracesFilterQuery(t *testing.T) { }}, }, want: "attributes_string['host'] ILIKE '%102.%' AND attributes_string['host'] NOT ILIKE '%103%' AND attributes_string['host'] ILIKE '102.' AND attributes_string['host'] NOT ILIKE '102' AND " + - "match(`path`, '/mypath') AND NOT match(`path`, '/health.*')", + "match(`attribute_string_path`, '/mypath') AND NOT match(`attribute_string_path`, '/health.*')", }, { name: "Test exists, nexists", @@ -241,32 +241,6 @@ func Test_buildTracesFilterQuery(t *testing.T) { } } -var handleEmptyValuesInGroupByData = []struct { - Name string - GroupBy []v3.AttributeKey - ExpectedFilter string -}{ - { - Name: "String type key", - GroupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, - ExpectedFilter: " AND has(stringTagMap, 'bytes')", - }, - { - Name: "fixed column type key", - GroupBy: []v3.AttributeKey{{Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, - ExpectedFilter: "", - }, - { - Name: "String, float64 and fixed column type key", - GroupBy: []v3.AttributeKey{ - {Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, - {Key: "count", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, - {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, - }, - ExpectedFilter: " AND has(stringTagMap, 'bytes') AND has(numberTagMap, 'count')", - }, -} - func Test_handleEmptyValuesInGroupBy(t *testing.T) { type args struct { groupBy []v3.AttributeKey @@ -383,7 +357,7 @@ func Test_buildTracesQuery(t *testing.T) { SelectColumns: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, }, }, - want: "SELECT timestamp as timestamp_datetime, spanID, traceID, `name` as `name`, `id` as `id` from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + want: "SELECT timestamp as timestamp_datetime, spanID, traceID, name as `name`, id as `id` from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) order by timestamp DESC", }, { @@ -430,7 +404,7 @@ func Test_buildTracesQuery(t *testing.T) { }, }, }, - want: "SELECT toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND " + + want: "SELECT toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND " + "(ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) having value > 10 order by value ASC", }, } @@ -519,7 +493,7 @@ func TestPrepareTracesQuery(t *testing.T) { GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, - want: "SELECT `serviceName` from (SELECT `serviceName` as `serviceName`, toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 " + + want: "SELECT `serviceName` from (SELECT serviceName as `serviceName`, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) group by `serviceName`) LIMIT 10", }, { @@ -540,7 +514,7 @@ func TestPrepareTracesQuery(t *testing.T) { GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, - want: "SELECT `serviceName` as `serviceName`, toFloat64(count(distinct(`name`))) as value from signoz_traces.distributed_signoz_index_v3 where " + + want: "SELECT serviceName as `serviceName`, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 where " + "(timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND (`serviceName`) GLOBAL IN (%s) group by `serviceName`", }, } diff --git a/pkg/query-service/rules/threshold_rule_test.go b/pkg/query-service/rules/threshold_rule_test.go index 852adac33d..30daf9de56 100644 --- a/pkg/query-service/rules/threshold_rule_test.go +++ b/pkg/query-service/rules/threshold_rule_test.go @@ -1648,7 +1648,7 @@ func TestThresholdRuleShiftBy(t *testing.T) { }, } - rule, err := NewThresholdRule("69", &postableRule, nil, nil, true) + rule, err := NewThresholdRule("69", &postableRule, nil, nil, true, true) if err != nil { assert.NoError(t, err) } From bb6f027b210c46543bf8a835a773dfbbf592c438 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 8 Nov 2024 13:13:18 +0530 Subject: [PATCH 18/41] fix: group by with filters --- pkg/query-service/app/traces/v4/query_builder.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index e7fbf83f4b..60711e09d4 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -227,15 +227,16 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. if err != nil { return "", err } + if filterSubQuery != "" { + filterSubQuery = " AND " + filterSubQuery + } emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(mq.GroupBy) if err != nil { return "", err } - filterSubQuery += emptyValuesInGroupByFilter - - if filterSubQuery != "" { - filterSubQuery = " AND " + filterSubQuery + if emptyValuesInGroupByFilter != "" { + filterSubQuery = " AND " + emptyValuesInGroupByFilter } resourceSubQuery, err := resource.BuildResourceSubQuery("signoz_traces", "distributed_traces_v3_resource", bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) From 80eda3c80572d5039c13763f52f82660ae2f86ff Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 8 Nov 2024 14:02:44 +0530 Subject: [PATCH 19/41] fix: add subquery --- pkg/query-service/app/traces/v4/query_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index 60711e09d4..8f05955d15 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -236,7 +236,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. return "", err } if emptyValuesInGroupByFilter != "" { - filterSubQuery = " AND " + emptyValuesInGroupByFilter + filterSubQuery = filterSubQuery + " AND " + emptyValuesInGroupByFilter } resourceSubQuery, err := resource.BuildResourceSubQuery("signoz_traces", "distributed_traces_v3_resource", bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) From b8436610974318c441fd6c5c144e4e86b44075b8 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 8 Nov 2024 14:13:31 +0530 Subject: [PATCH 20/41] fix: trigger builde --- pkg/query-service/app/traces/v4/query_builder.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index 8f05955d15..abeda49bb5 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -249,7 +249,6 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. } // timerange will be sent in epoch millisecond - selectLabels := getSelectLabels(mq.GroupBy) if selectLabels != "" { selectLabels = selectLabels + "," From 549485bbe937c39defc3ebdcc125e047a9b966d9 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Sun, 10 Nov 2024 17:18:49 +0530 Subject: [PATCH 21/41] fix: update pagination logic and few ch column names --- .../app/clickhouseReader/reader.go | 4 +-- pkg/query-service/app/querier/querier.go | 30 +++++++++++++------ .../app/traces/v4/query_builder.go | 1 - .../app/traces/v4/query_builder_test.go | 2 +- pkg/query-service/model/response.go | 8 ++--- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 0b9a41d549..7c2946d383 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -1709,7 +1709,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea var traceSummary model.TraceSummary summaryQuery := fmt.Sprintf("SELECT * from %s.%s WHERE traceID=$1", r.TraceDB, r.traceSummaryTable) - err := r.db.QueryRow(ctx, summaryQuery, params.TraceID).Scan(&traceSummary.TraceID, &traceSummary.FirstReported, &traceSummary.LastReported, &traceSummary.NumSpans) + err := r.db.QueryRow(ctx, summaryQuery, params.TraceID).Scan(&traceSummary.TraceID, &traceSummary.Start, &traceSummary.End, &traceSummary.NumSpans) if err != nil { if err == sql.ErrNoRows { return &searchSpansResult, nil @@ -1747,7 +1747,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea start := time.Now() - err = r.db.Select(ctx, &searchScanResponses, query, params.TraceID, strconv.FormatInt(traceSummary.FirstReported.Unix()-1800, 10), strconv.FormatInt(traceSummary.LastReported.Unix(), 10)) + err = r.db.Select(ctx, &searchScanResponses, query, params.TraceID, strconv.FormatInt(traceSummary.Start.Unix()-1800, 10), strconv.FormatInt(traceSummary.End.Unix(), 10)) zap.L().Info(query) diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index 6689f85241..993c0ff0b4 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -354,15 +354,27 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR // append a filter to the params if len(data) > 0 { - params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ - Key: v3.AttributeKey{ - Key: "id", - IsColumn: true, - DataType: "string", - }, - Operator: v3.FilterOperatorLessThan, - Value: data[len(data)-1].Data["id"], - }) + if params.CompositeQuery.BuilderQueries[qName].DataSource == v3.DataSourceLogs { + params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "id", + IsColumn: true, + DataType: "string", + }, + Operator: v3.FilterOperatorLessThan, + Value: data[len(data)-1].Data["id"], + }) + } else { + // for traces setting offset = 0 works + // eg - + // 1)--- searching 100 logs in between t1, t10, t100 with offset 0 + // if 100 logs are there in t1 to t10 then 100 will return immediately. + // if 10 logs are there in t1 to t10 then we get 10, set offset to 0 and search in the next timerange of t10 to t100. + // 1)--- searching 100 logs in between t1, t10, t100 with offset 100 + // It will have offset = 0 till 100 logs are found in one of the timerange tx to tx-1 + // If it finds <100 in tx to tx-1 then it will set offset = 0 and search in the next timerange of tx-1 to tx-2 + params.CompositeQuery.BuilderQueries[qName].Offset = 0 + } } if uint64(len(data)) >= pageSize { diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index abeda49bb5..b9964de037 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -273,7 +273,6 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. return "", fmt.Errorf("select columns cannot be empty for panelType %s", panelType) } // add it to the select labels - mq.SelectColumns = append(mq.SelectColumns, v3.AttributeKey{Key: "id", IsColumn: true}) selectLabels = getSelectLabels(mq.SelectColumns) queryNoOpTmpl := fmt.Sprintf("SELECT timestamp as timestamp_datetime, spanID, traceID,%s ", selectLabels) + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_V3 + " where %s %s" + "%s" query = fmt.Sprintf(queryNoOpTmpl, timeFilter, filterSubQuery, orderBy) diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index 6094ac9892..1f15dad4e1 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -357,7 +357,7 @@ func Test_buildTracesQuery(t *testing.T) { SelectColumns: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, }, }, - want: "SELECT timestamp as timestamp_datetime, spanID, traceID, name as `name`, id as `id` from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + want: "SELECT timestamp as timestamp_datetime, spanID, traceID, name as `name` from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) order by timestamp DESC", }, { diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index e0fbfc5860..de49e29238 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -289,10 +289,10 @@ type SearchSpanResponseItemV2 struct { } type TraceSummary struct { - TraceID string `json:"traceID" ch:"traceID"` - FirstReported time.Time `json:"firstReported" ch:"first_reported"` - LastReported time.Time `json:"lastReported" ch:"last_reported"` - NumSpans uint64 `json:"numSpans" ch:"num_spans"` + TraceID string `json:"traceId" ch:"trace_id"` + Start time.Time `json:"start" ch:"start"` + End time.Time `json:"end" ch:"end"` + NumSpans uint64 `json:"numSpans" ch:"num_spans"` } type OtelSpanRef struct { From 16d538e1bade0d14528c5e9017bec63240134ac6 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 13 Nov 2024 18:00:15 +0530 Subject: [PATCH 22/41] fix: update qb --- .../app/traces/v4/query_builder.go | 43 +-- .../app/traces/v4/query_builder_test.go | 260 ++++++++++++++---- pkg/query-service/constants/constants.go | 1 + 3 files changed, 212 insertions(+), 92 deletions(-) diff --git a/pkg/query-service/app/traces/v4/query_builder.go b/pkg/query-service/app/traces/v4/query_builder.go index b9964de037..824921b7e8 100644 --- a/pkg/query-service/app/traces/v4/query_builder.go +++ b/pkg/query-service/app/traces/v4/query_builder.go @@ -11,10 +11,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/utils" ) -type Options struct { - GraphLimitQtype string - PreferRPM bool -} +const NANOSECOND = 1000000000 var tracesOperatorMappingV3 = map[v3.FilterOperator]string{ v3.FilterOperatorIn: "IN", @@ -32,7 +29,7 @@ var tracesOperatorMappingV3 = map[v3.FilterOperator]string{ v3.FilterOperatorContains: "ILIKE", v3.FilterOperatorNotContains: "NOT ILIKE", v3.FilterOperatorExists: "mapContains(%s, '%s')", - v3.FilterOperatorNotExists: "NOT has(%s%s, '%s')", + v3.FilterOperatorNotExists: "NOT mapContains(%s, '%s')", } func getClickHouseTracesColumnType(columnType v3.AttributeKeyType) string { @@ -53,23 +50,17 @@ func getClickHouseTracesColumnDataType(columnDataType v3.AttributeKeyDataType) s } func getColumnName(key v3.AttributeKey) string { + // if key present in static return as it is + if _, ok := constants.StaticFieldsTraces[key.Key]; ok { + return key.Key + } + if !key.IsColumn { keyType := getClickHouseTracesColumnType(key.Type) keyDType := getClickHouseTracesColumnDataType(key.DataType) return fmt.Sprintf("%s_%s['%s']", keyType, keyDType, key.Key) } - // check if it is a static field - if key.Type == v3.AttributeKeyTypeUnspecified { - // name is the column name - return key.Key - } - - // if key present in static return as it is - if _, ok := constants.StaticFieldsTraces[key.Key]; ok { - return key.Key - } - return "`" + utils.GetClickhouseColumnNameV2(string(key.Type), string(key.DataType), key.Key) + "`" } @@ -112,7 +103,8 @@ func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) { if operator, ok := tracesOperatorMappingV3[item.Operator]; ok { switch item.Operator { case v3.FilterOperatorContains, v3.FilterOperatorNotContains: - val = utils.QuoteEscapedString(fmt.Sprintf("%v", item.Value)) + // we also want to treat %, _ as literals for contains + val := utils.QuoteEscapedStringForContains(fmt.Sprintf("%s", item.Value), false) conditions = append(conditions, fmt.Sprintf("%s %s '%%%s%%'", columnName, operator, val)) case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex: conditions = append(conditions, fmt.Sprintf(operator, columnName, fmtVal)) @@ -144,6 +136,8 @@ func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) { } func handleEmptyValuesInGroupBy(groupBy []v3.AttributeKey) (string, error) { + // TODO(nitya): in future when we support user based mat column handle them + // skipping now as we don't support creating them filterItems := []v3.FilterItem{} if len(groupBy) != 0 { for _, item := range groupBy { @@ -165,8 +159,6 @@ func handleEmptyValuesInGroupBy(groupBy []v3.AttributeKey) (string, error) { return "", nil } -const NANOSECOND = 1000000000 - // orderBy returns a string of comma separated tags for order by clause // if there are remaining items which are not present in tags they are also added // if the order is not specified, it defaults to ASC @@ -181,11 +173,7 @@ func orderBy(panelType v3.PanelType, items []v3.OrderBy, tagLookup map[string]st } else if panelType == v3.PanelTypeList { attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn} name := getColumnName(attr) - if item.IsColumn { - orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) - } else { - orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) - } + orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) } } @@ -200,7 +188,6 @@ func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags [] orderByArray := orderBy(panelType, items, tagLookup) - // TODO: check this with logs if len(orderByArray) == 0 { if panelType == v3.PanelTypeList { orderByArray = append(orderByArray, constants.TIMESTAMP+" DESC") @@ -304,8 +291,6 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. } else if panelType == v3.PanelTypeTable { queryTmpl = "SELECT " - // step or aggregate interval is whole time period in case of table panel - step = (tracesEnd - tracesStart) / 1000000000 } else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue { // Select the aggregate value for interval queryTmpl = @@ -368,7 +353,6 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. filterSubQuery = fmt.Sprintf("%s AND %s", filterSubQuery, subQuery) } } else { - // columnType, columnDataType := getClickhouseTracesColumnDataTypeAndType(mq.AggregateAttribute) column := getColumnName(mq.AggregateAttribute) filterSubQuery = fmt.Sprintf("%s AND has(%s, '%s')", filterSubQuery, column, mq.AggregateAttribute.Key) } @@ -380,9 +364,6 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3. op := fmt.Sprintf("toFloat64(count(distinct(%s)))", aggregationKey) query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy) return query, nil - // case v3.AggregateOperatorNoOp: - - // return query, nil default: return "", fmt.Errorf("unsupported aggregate operator %s", mq.AggregateOperator) } diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index 1f15dad4e1..449fefc039 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -110,6 +110,13 @@ func Test_getColumnName(t *testing.T) { }, want: "`attribute_string_data`", }, + { + name: "static column", + args: args{ + key: v3.AttributeKey{Key: "spanKind", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + }, + want: "spanKind", + }, { name: "missing meta", args: args{ @@ -147,11 +154,11 @@ func Test_getSelectLabels(t *testing.T) { name: "multiple group by", args: args{ groupBy: []v3.AttributeKey{ - {Key: "user_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, // static col {Key: "service_name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource, IsColumn: true}, }, }, - want: " attributes_string['user_name'] as `user_name`, `resource_string_service_name` as `service_name`", + want: " name as `name`, `resource_string_service_name` as `service_name`", }, } for _, tt := range tests { @@ -173,6 +180,15 @@ func Test_buildTracesFilterQuery(t *testing.T) { want string wantErr bool }{ + { + name: "Test ignore resource", + args: args{ + fs: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "service.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: []interface{}{"service"}, Operator: v3.FilterOperatorIn}, + }, + }}, + want: "", + }, { name: "Test buildTracesFilterQuery in, nin", args: args{ @@ -180,9 +196,12 @@ func Test_buildTracesFilterQuery(t *testing.T) { {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{"GET", "POST"}, Operator: v3.FilterOperatorIn}, {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{"PUT"}, Operator: v3.FilterOperatorNotIn}, {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: []interface{}{"server"}, Operator: v3.FilterOperatorNotIn}, + {Key: v3.AttributeKey{Key: "status.code", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{200}, Operator: v3.FilterOperatorNotIn}, + {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{100.0}, Operator: v3.FilterOperatorIn}, + {Key: v3.AttributeKey{Key: "isDone", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, Value: []interface{}{true}, Operator: v3.FilterOperatorIn}, }}, }, - want: "attributes_string['method'] IN ['GET','POST'] AND attributes_string['method'] NOT IN ['PUT']", + want: "attributes_string['method'] IN ['GET','POST'] AND attributes_string['method'] NOT IN ['PUT'] AND attributes_number['status.code'] NOT IN [200] AND attributes_number['duration'] IN [100] AND attributes_bool['isDone'] IN [true]", wantErr: false, }, { @@ -193,27 +212,27 @@ func Test_buildTracesFilterQuery(t *testing.T) { {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 100, Operator: v3.FilterOperatorNotEqual}, {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: v3.FilterOperatorGreaterThan}, {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 200, Operator: v3.FilterOperatorLessThan}, - {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: v3.FilterOperatorGreaterThanOrEq}, - {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 200, Operator: v3.FilterOperatorLessThanOrEq}, + {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag}, Value: 10.0, Operator: v3.FilterOperatorGreaterThanOrEq}, + {Key: v3.AttributeKey{Key: "duration_str", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "200", Operator: v3.FilterOperatorLessThanOrEq}, }}, }, want: "attributes_number['duration'] = 102 AND attributes_number['duration'] != 100 AND attributes_number['duration'] > 10 AND attributes_number['duration'] < 200" + - " AND attributes_number['duration'] >= 10 AND attributes_number['duration'] <= 200", + " AND attributes_number['duration'] >= 10.000000 AND attributes_string['duration_str'] <= '200'", wantErr: false, }, { name: "Test contains, ncontains, like, nlike, regex, nregex", args: args{ fs: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ - {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.", Operator: v3.FilterOperatorContains}, - {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "103", Operator: v3.FilterOperatorNotContains}, + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.%", Operator: v3.FilterOperatorContains}, + {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "103_", Operator: v3.FilterOperatorNotContains}, {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102.", Operator: v3.FilterOperatorLike}, {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "102", Operator: v3.FilterOperatorNotLike}, {Key: v3.AttributeKey{Key: "path", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "/mypath", Operator: v3.FilterOperatorRegex}, {Key: v3.AttributeKey{Key: "path", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Value: "/health.*", Operator: v3.FilterOperatorNotRegex}, }}, }, - want: "attributes_string['host'] ILIKE '%102.%' AND attributes_string['host'] NOT ILIKE '%103%' AND attributes_string['host'] ILIKE '102.' AND attributes_string['host'] NOT ILIKE '102' AND " + + want: "attributes_string['host'] ILIKE '%102.\\%%' AND attributes_string['host'] NOT ILIKE '%103\\_%' AND attributes_string['host'] ILIKE '102.' AND attributes_string['host'] NOT ILIKE '102' AND " + "match(`attribute_string_path`, '/mypath') AND NOT match(`attribute_string_path`, '/health.*')", }, { @@ -221,10 +240,13 @@ func Test_buildTracesFilterQuery(t *testing.T) { args: args{ fs: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ {Key: v3.AttributeKey{Key: "host", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: v3.FilterOperatorExists}, + {Key: v3.AttributeKey{Key: "duration", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Operator: v3.FilterOperatorExists}, + {Key: v3.AttributeKey{Key: "isDone", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, Operator: v3.FilterOperatorNotExists}, + {Key: v3.AttributeKey{Key: "host1", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: v3.FilterOperatorNotExists}, {Key: v3.AttributeKey{Key: "path", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Operator: v3.FilterOperatorNotExists}, }}, }, - want: "mapContains(attributes_string, 'host') AND path = ''", + want: "mapContains(attributes_string, 'host') AND mapContains(attributes_number, 'duration') AND NOT mapContains(attributes_bool, 'isDone') AND NOT mapContains(attributes_string, 'host1') AND path = ''", }, } for _, tt := range tests { @@ -282,6 +304,54 @@ func Test_handleEmptyValuesInGroupBy(t *testing.T) { } } +func Test_orderByAttributeKeyTags(t *testing.T) { + type args struct { + panelType v3.PanelType + items []v3.OrderBy + tags []v3.AttributeKey + } + tests := []struct { + name string + args args + want string + }{ + { + name: "test", + args: args{ + panelType: v3.PanelTypeGraph, + items: []v3.OrderBy{{ColumnName: "name", Order: "ASC"}}, + tags: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + want: "`name` ASC", + }, + { + name: "order by value", + args: args{ + panelType: v3.PanelTypeGraph, + items: []v3.OrderBy{{ColumnName: "name", Order: "ASC"}, {ColumnName: constants.SigNozOrderByValue, Order: "DESC"}}, + tags: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + want: "`name` ASC,value DESC", + }, + { + name: "test", + args: args{ + panelType: v3.PanelTypeList, + items: []v3.OrderBy{{ColumnName: "status", Order: "DESC", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {ColumnName: "route", Order: "DESC", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, + }, + want: "attributes_string['status'] DESC,`attribute_string_route` DESC", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := orderByAttributeKeyTags(tt.args.panelType, tt.args.items, tt.args.tags); got != tt.want { + t.Errorf("orderByAttributeKeyTags() = %v, want %v", got, tt.want) + } + }) + } +} + func Test_buildTracesQuery(t *testing.T) { type args struct { start int64 @@ -307,16 +377,22 @@ func Test_buildTracesQuery(t *testing.T) { mq: &v3.BuilderQuery{ AggregateOperator: v3.AggregateOperatorCount, Filters: &v3.FilterSet{ - Items: []v3.FilterItem{}, + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + Value: 100, + Operator: v3.FilterOperatorEqual, + }, + }, }, GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, OrderBy: []v3.OrderBy{ {ColumnName: "http.method", Order: "ASC"}}, }, }, - want: "SELECT attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 " + - "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + - "AND mapContains(attributes_string, 'http.method') group by `http.method` order by `http.method` ASC", + want: "SELECT attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') " + + "group by `http.method` order by `http.method` ASC", }, { name: "Test buildTracesQuery", @@ -347,6 +423,22 @@ func Test_buildTracesQuery(t *testing.T) { }, { name: "test noop list view", + args: args{ + panelType: v3.PanelTypeList, + start: 1680066360726210000, + end: 1680066458000000000, + mq: &v3.BuilderQuery{ + AggregateOperator: v3.AggregateOperatorNoOp, + Filters: &v3.FilterSet{}, + SelectColumns: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, + OrderBy: []v3.OrderBy{{ColumnName: "timestamp", Order: "ASC"}}, + }, + }, + want: "SELECT timestamp as timestamp_datetime, spanID, traceID, name as `name` from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + + "AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) order by timestamp ASC", + }, + { + name: "test noop list view-without ts", args: args{ panelType: v3.PanelTypeList, start: 1680066360726210000, @@ -422,45 +514,6 @@ func Test_buildTracesQuery(t *testing.T) { } } -func Test_orderByAttributeKeyTags(t *testing.T) { - type args struct { - panelType v3.PanelType - items []v3.OrderBy - tags []v3.AttributeKey - } - tests := []struct { - name string - args args - want string - }{ - { - name: "test", - args: args{ - panelType: v3.PanelTypeTrace, - items: []v3.OrderBy{{ColumnName: "name", Order: "ASC"}}, - tags: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, - }, - want: "`name` ASC", - }, - { - name: "test", - args: args{ - panelType: v3.PanelTypeList, - items: []v3.OrderBy{{ColumnName: "name", Order: "DESC"}}, - tags: []v3.AttributeKey{{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, - }, - want: "`name` DESC", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := orderByAttributeKeyTags(tt.args.panelType, tt.args.items, tt.args.tags); got != tt.want { - t.Errorf("orderByAttributeKeyTags() = %v, want %v", got, tt.want) - } - }) - } -} - func TestPrepareTracesQuery(t *testing.T) { type args struct { start int64 @@ -485,16 +538,17 @@ func TestPrepareTracesQuery(t *testing.T) { StepInterval: 60, AggregateOperator: v3.AggregateOperatorCountDistinct, Filters: &v3.FilterSet{}, - AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, - GroupBy: []v3.AttributeKey{{Key: "serviceName", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + GroupBy: []v3.AttributeKey{{Key: "function", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, Limit: 10, + OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "DESC"}}, }, options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, - want: "SELECT `serviceName` from (SELECT serviceName as `serviceName`, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 " + - "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) group by `serviceName`) LIMIT 10", + want: "SELECT `function` from (SELECT attributes_string['function'] as `function`, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND mapContains(attributes_string, 'function') group by `function` order by value DESC) LIMIT 10", }, { name: "test with limit - second", @@ -506,16 +560,100 @@ func TestPrepareTracesQuery(t *testing.T) { StepInterval: 60, AggregateOperator: v3.AggregateOperatorCountDistinct, Filters: &v3.FilterSet{}, - AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, - GroupBy: []v3.AttributeKey{{Key: "serviceName", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + GroupBy: []v3.AttributeKey{{Key: "function", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "DESC"}}, Limit: 10, }, options: v3.QBOptions{ GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, - want: "SELECT serviceName as `serviceName`, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 where " + - "(timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND (`serviceName`) GLOBAL IN (%s) group by `serviceName`", + want: "SELECT attributes_string['function'] as `function`, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 where " + + "(timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND mapContains(attributes_string, 'function') AND (`function`) GLOBAL IN (%s) group by `function` order by value DESC", + }, + { + name: "test with limit with resources- first", + args: args{ + start: 1680066360726210000, + end: 1680066458000000000, + panelType: v3.PanelTypeTable, + mq: &v3.BuilderQuery{ + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{Key: "line", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, + Value: 100, + Operator: v3.FilterOperatorEqual, + }, + { + Key: v3.AttributeKey{Key: "hostname", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, + Value: "server1", + Operator: v3.FilterOperatorEqual, + }, + }, + }, + AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + GroupBy: []v3.AttributeKey{ + {Key: "function", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "service.name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, + }, + Limit: 10, + OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "DESC"}}, + }, + options: v3.QBOptions{ + GraphLimitQtype: constants.FirstQueryGraphLimit, + }, + }, + want: "SELECT `function`,`service.name` from (SELECT `attribute_string_function` as `function`, `resource_string_service$$name` as `service.name`, toFloat64(count(distinct(name))) as value " + + "from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) " + + "AND attributes_number['line'] = 100 AND (resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE " + + "(seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) AND simpleJSONExtractString(labels, 'hostname') = 'server1' AND labels like '%hostname%server1%' AND " + + "( (simpleJSONHas(labels, 'service.name') AND labels like '%service.name%') ))) group by `function`,`service.name` order by value DESC) LIMIT 10", + }, + { + name: "test with limit with resources - second", + args: args{ + start: 1680066360726210000, + end: 1680066458000000000, + panelType: v3.PanelTypeTable, + mq: &v3.BuilderQuery{ + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{Key: "line", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, + Value: 100, + Operator: v3.FilterOperatorEqual, + }, + { + Key: v3.AttributeKey{Key: "hostname", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, + Value: "server1", + Operator: v3.FilterOperatorEqual, + }, + }, + }, + AggregateAttribute: v3.AttributeKey{Key: "name", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + GroupBy: []v3.AttributeKey{ + {Key: "function", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "serviceName", IsColumn: true, DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + }, + OrderBy: []v3.OrderBy{{ColumnName: "#SIGNOZ_VALUE", Order: "DESC"}}, + Limit: 10, + }, + options: v3.QBOptions{ + GraphLimitQtype: constants.SecondQueryGraphLimit, + }, + }, + want: "SELECT `attribute_string_function` as `function`, serviceName as `serviceName`, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 " + + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_number['line'] = 100 " + + "AND (resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) " + + "AND simpleJSONExtractString(labels, 'hostname') = 'server1' AND labels like '%hostname%server1%')) AND (`function`,`serviceName`) GLOBAL IN (%s) group by `function`,`serviceName` order by value DESC", }, } diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 1f0371e4a7..af2c5729cb 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -434,6 +434,7 @@ var SpanRenderLimitStr = GetOrDefaultEnv("SPAN_RENDER_LIMIT", "2500") var MaxSpansInTraceStr = GetOrDefaultEnv("MAX_SPANS_IN_TRACE", "250000") var StaticFieldsTraces = map[string]v3.AttributeKey{ + "timestamp": {}, "traceID": { Key: "traceID", DataType: v3.AttributeKeyDataTypeString, From 471bd684c8d361a803f59bd566c85e464029a1be Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 13 Nov 2024 20:06:34 +0530 Subject: [PATCH 23/41] fix: add tests --- .../app/traces/v4/query_builder_test.go | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/pkg/query-service/app/traces/v4/query_builder_test.go b/pkg/query-service/app/traces/v4/query_builder_test.go index 449fefc039..59fe1998ef 100644 --- a/pkg/query-service/app/traces/v4/query_builder_test.go +++ b/pkg/query-service/app/traces/v4/query_builder_test.go @@ -342,6 +342,42 @@ func Test_orderByAttributeKeyTags(t *testing.T) { }, want: "attributes_string['status'] DESC,`attribute_string_route` DESC", }, + { + name: "ignore order by in table panel", + args: args{ + panelType: v3.PanelTypeTable, + items: []v3.OrderBy{{ColumnName: "timestamp", Order: "DESC"}}, + tags: []v3.AttributeKey{}, + }, + want: "", + }, + { + name: "add default order by ts for list panel", + args: args{ + panelType: v3.PanelTypeList, + items: []v3.OrderBy{}, + tags: []v3.AttributeKey{}, + }, + want: "timestamp DESC", + }, + { + name: "add default order by value for graph panel", + args: args{ + panelType: v3.PanelTypeGraph, + items: []v3.OrderBy{}, + tags: []v3.AttributeKey{}, + }, + want: "value DESC", + }, + { + name: "don't add default order by for table panel", + args: args{ + panelType: v3.PanelTypeTable, + items: []v3.OrderBy{}, + tags: []v3.AttributeKey{}, + }, + want: "", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From f64285b89d77d82b92d406f359db7233dfc9a555 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 14 Nov 2024 16:22:21 +0530 Subject: [PATCH 24/41] feat: minor fixes --- ee/query-service/app/server.go | 1 - ee/query-service/rules/manager.go | 1 + pkg/query-service/app/querier/querier.go | 12 ++++++++--- pkg/query-service/app/querier/v2/querier.go | 12 ++++++++--- pkg/query-service/rules/manager.go | 22 +++++++++++--------- pkg/query-service/rules/test_notification.go | 1 + pkg/query-service/utils/logs.go | 2 +- pkg/query-service/utils/logs_test.go | 4 ++-- 8 files changed, 35 insertions(+), 20 deletions(-) diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index feae157adf..b58a90b9c4 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -774,7 +774,6 @@ func makeRulesManager( PrepareTaskFunc: rules.PrepareTaskFunc, UseLogsNewSchema: useLogsNewSchema, UseTraceNewSchema: useTraceNewSchema, - PrepareTaskFunc: rules.PrepareTaskFunc, PrepareTestRuleFunc: rules.TestNotification, } diff --git a/ee/query-service/rules/manager.go b/ee/query-service/rules/manager.go index 7214f338ca..00e0882f36 100644 --- a/ee/query-service/rules/manager.go +++ b/ee/query-service/rules/manager.go @@ -123,6 +123,7 @@ func TestNotification(opts baserules.PrepareTestRuleOptions) (int, *basemodel.Ap opts.FF, opts.Reader, opts.UseLogsNewSchema, + opts.UseTraceNewSchema, baserules.WithSendAlways(), baserules.WithSendUnmatched(), ) diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index 993c0ff0b4..5314c4c2ad 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -389,18 +389,24 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR } func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3) ([]*v3.Result, map[string]error, error) { - // List query has support for only one query. - if q.UseLogsNewSchema && params.CompositeQuery != nil && + // List query has support for only one query + // we are skipping for PanelTypeTrace as it has a custom order by regardless of what's in the payload + if params.CompositeQuery != nil && len(params.CompositeQuery.BuilderQueries) == 1 && params.CompositeQuery.PanelType != v3.PanelTypeTrace { for _, v := range params.CompositeQuery.BuilderQueries { + if (v.DataSource == v3.DataSourceLogs && !q.UseLogsNewSchema) || + (v.DataSource == v3.DataSourceTraces && !q.UseTraceNewSchema) { + break + } + // only allow of logs queries with timestamp ordering desc // TODO(nitya): allow for timestamp asc if (v.DataSource == v3.DataSourceLogs || v.DataSource == v3.DataSourceTraces) && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { - startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) + startEndArr := utils.GetListTsRanges(params.Start, params.End) return q.runWindowBasedListQuery(ctx, params, startEndArr) } } diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index a373ae1589..de92b7d465 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -377,18 +377,24 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR } func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3) ([]*v3.Result, map[string]error, error) { - // List query has support for only one query. - if q.UseLogsNewSchema && params.CompositeQuery != nil && + // List query has support for only one query + // we are skipping for PanelTypeTrace as it has a custom order by regardless of what's in the payload + if params.CompositeQuery != nil && len(params.CompositeQuery.BuilderQueries) == 1 && params.CompositeQuery.PanelType != v3.PanelTypeTrace { for _, v := range params.CompositeQuery.BuilderQueries { + if (v.DataSource == v3.DataSourceLogs && !q.UseLogsNewSchema) || + (v.DataSource == v3.DataSourceTraces && !q.UseTraceNewSchema) { + break + } + // only allow of logs queries with timestamp ordering desc // TODO(nitya): allow for timestamp asc if (v.DataSource == v3.DataSourceLogs || v.DataSource == v3.DataSourceTraces) && len(v.OrderBy) == 1 && v.OrderBy[0].ColumnName == "timestamp" && v.OrderBy[0].Order == "desc" { - startEndArr := utils.GetLogsListTsRanges(params.Start, params.End) + startEndArr := utils.GetListTsRanges(params.Start, params.End) return q.runWindowBasedListQuery(ctx, params, startEndArr) } } diff --git a/pkg/query-service/rules/manager.go b/pkg/query-service/rules/manager.go index 07a8e4855b..84e055cfa0 100644 --- a/pkg/query-service/rules/manager.go +++ b/pkg/query-service/rules/manager.go @@ -49,7 +49,8 @@ type PrepareTestRuleOptions struct { ManagerOpts *ManagerOptions NotifyFunc NotifyFunc - UseLogsNewSchema bool + UseLogsNewSchema bool + UseTraceNewSchema bool } const taskNamesuffix = "webAppEditor" @@ -814,15 +815,16 @@ func (m *Manager) TestNotification(ctx context.Context, ruleStr string) (int, *m } alertCount, apiErr := m.prepareTestRuleFunc(PrepareTestRuleOptions{ - Rule: parsedRule, - RuleDB: m.ruleDB, - Logger: m.logger, - Reader: m.reader, - Cache: m.cache, - FF: m.featureFlags, - ManagerOpts: m.opts, - NotifyFunc: m.prepareNotifyFunc(), - UseLogsNewSchema: m.opts.UseLogsNewSchema, + Rule: parsedRule, + RuleDB: m.ruleDB, + Logger: m.logger, + Reader: m.reader, + Cache: m.cache, + FF: m.featureFlags, + ManagerOpts: m.opts, + NotifyFunc: m.prepareNotifyFunc(), + UseLogsNewSchema: m.opts.UseLogsNewSchema, + UseTraceNewSchema: m.opts.UseTraceNewSchema, }) return alertCount, apiErr diff --git a/pkg/query-service/rules/test_notification.go b/pkg/query-service/rules/test_notification.go index 37fb2e5f1b..e30b7db94f 100644 --- a/pkg/query-service/rules/test_notification.go +++ b/pkg/query-service/rules/test_notification.go @@ -49,6 +49,7 @@ func defaultTestNotification(opts PrepareTestRuleOptions) (int, *model.ApiError) opts.FF, opts.Reader, opts.UseLogsNewSchema, + opts.UseTraceNewSchema, WithSendAlways(), WithSendUnmatched(), ) diff --git a/pkg/query-service/utils/logs.go b/pkg/query-service/utils/logs.go index 11129ae3d8..3dc4a50f48 100644 --- a/pkg/query-service/utils/logs.go +++ b/pkg/query-service/utils/logs.go @@ -9,7 +9,7 @@ type LogsListTsRange struct { End int64 } -func GetLogsListTsRanges(start, end int64) []LogsListTsRange { +func GetListTsRanges(start, end int64) []LogsListTsRange { startNano := GetEpochNanoSecs(start) endNano := GetEpochNanoSecs(end) result := []LogsListTsRange{} diff --git a/pkg/query-service/utils/logs_test.go b/pkg/query-service/utils/logs_test.go index 4a69534f80..dc0c9663fa 100644 --- a/pkg/query-service/utils/logs_test.go +++ b/pkg/query-service/utils/logs_test.go @@ -7,7 +7,7 @@ import ( v3 "go.signoz.io/signoz/pkg/query-service/model/v3" ) -func TestLogsListTsRange(t *testing.T) { +func TestListTsRange(t *testing.T) { startEndData := []struct { name string start int64 @@ -46,7 +46,7 @@ func TestLogsListTsRange(t *testing.T) { } for _, test := range startEndData { - res := GetLogsListTsRanges(test.start, test.end) + res := GetListTsRanges(test.start, test.end) for i, v := range res { if test.res[i].Start != v.Start || test.res[i].End != v.End { t.Errorf("expected range was %v - %v, got %v - %v", v.Start, v.End, test.res[i].Start, test.res[i].End) From 252e0b698e0a588f31d03bf617cb9e45ecb61785 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 14 Nov 2024 17:40:56 +0530 Subject: [PATCH 25/41] fix: update pagination logic --- pkg/query-service/app/querier/querier.go | 40 ++++++++++------- pkg/query-service/app/querier/v2/querier.go | 49 ++++++++++++++++----- 2 files changed, 62 insertions(+), 27 deletions(-) diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index 5314c4c2ad..d9d6e14129 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -321,11 +321,13 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR res := make([]*v3.Result, 0) qName := "" pageSize := uint64(0) + limit := uint64(0) // se we are considering only one query for name, v := range params.CompositeQuery.BuilderQueries { qName = name pageSize = v.PageSize + limit = v.Limit } data := []*v3.Row{} @@ -352,9 +354,8 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR data = append(data, rowList...) } - // append a filter to the params - if len(data) > 0 { - if params.CompositeQuery.BuilderQueries[qName].DataSource == v3.DataSourceLogs { + if params.CompositeQuery.BuilderQueries[qName].DataSource == v3.DataSourceLogs { + if len(data) > 0 { params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ Key: v3.AttributeKey{ Key: "id", @@ -364,21 +365,30 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR Operator: v3.FilterOperatorLessThan, Value: data[len(data)-1].Data["id"], }) - } else { - // for traces setting offset = 0 works - // eg - - // 1)--- searching 100 logs in between t1, t10, t100 with offset 0 - // if 100 logs are there in t1 to t10 then 100 will return immediately. - // if 10 logs are there in t1 to t10 then we get 10, set offset to 0 and search in the next timerange of t10 to t100. - // 1)--- searching 100 logs in between t1, t10, t100 with offset 100 - // It will have offset = 0 till 100 logs are found in one of the timerange tx to tx-1 - // If it finds <100 in tx to tx-1 then it will set offset = 0 and search in the next timerange of tx-1 to tx-2 + } + + if uint64(len(data)) >= pageSize { + break + } + } else { + // we are updating the offset and limit based on the number of traces we have found in the current timerange + // eg - + // 1)offset = 0, limit = 100, tsRanges = [t1, t10], [t10, 20], [t20, t30] + // if 100 traces are there in [t1, t10] then 100 will return immediately. + // if 10 traces are there in [t1, t10] then we get 10, set offset to 0 and limit to 90, search in the next timerange of [t10, 20] + // + // 2) offset = 50, limit = 100, tsRanges = [t1, t10], [t10, 20], [t20, t30] + // If we find 100 traces in [t1, t10] then we return immediately + // If we finds 50 in [t1, t10] then it will set offset = 0 and limit = 50 and search in the next timerange of [t10, 20] + // if we don't find any trace in [t1, t10], then we search in [t10, 20] with offset=50, limit=100 + if len(data) > 0 { params.CompositeQuery.BuilderQueries[qName].Offset = 0 + params.CompositeQuery.BuilderQueries[qName].Limit = limit - uint64(len(data)) } - } - if uint64(len(data)) >= pageSize { - break + if uint64(len(data)) >= limit { + break + } } } res = append(res, &v3.Result{ diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index de92b7d465..06c96a4a0e 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -321,11 +321,13 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR res := make([]*v3.Result, 0) qName := "" pageSize := uint64(0) + limit := uint64(0) // se we are considering only one query for name, v := range params.CompositeQuery.BuilderQueries { qName = name pageSize = v.PageSize + limit = v.Limit } data := []*v3.Row{} @@ -352,17 +354,41 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR data = append(data, rowList...) } - // append a filter to the params - if len(data) > 0 { - params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ - Key: v3.AttributeKey{ - Key: "id", - IsColumn: true, - DataType: "string", - }, - Operator: v3.FilterOperatorLessThan, - Value: data[len(data)-1].Data["id"], - }) + if params.CompositeQuery.BuilderQueries[qName].DataSource == v3.DataSourceLogs { + if len(data) > 0 { + params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ + Key: v3.AttributeKey{ + Key: "id", + IsColumn: true, + DataType: "string", + }, + Operator: v3.FilterOperatorLessThan, + Value: data[len(data)-1].Data["id"], + }) + } + + if uint64(len(data)) >= pageSize { + break + } + } else { + // we are updating the offset and limit based on the number of traces we have found in the current timerange + // eg - + // 1)offset = 0, limit = 100, tsRanges = [t1, t10], [t10, 20], [t20, t30] + // if 100 traces are there in [t1, t10] then 100 will return immediately. + // if 10 traces are there in [t1, t10] then we get 10, set offset to 0 and limit to 90, search in the next timerange of [t10, 20] + // + // 2) offset = 50, limit = 100, tsRanges = [t1, t10], [t10, 20], [t20, t30] + // If we find 100 traces in [t1, t10] then we return immediately + // If we finds 50 in [t1, t10] then it will set offset = 0 and limit = 50 and search in the next timerange of [t10, 20] + // if we don't find any trace in [t1, t10], then we search in [t10, 20] with offset=50, limit=100 + if len(data) > 0 { + params.CompositeQuery.BuilderQueries[qName].Offset = 0 + params.CompositeQuery.BuilderQueries[qName].Limit = limit - uint64(len(data)) + } + + if uint64(len(data)) >= limit { + break + } } if uint64(len(data)) >= pageSize { @@ -375,7 +401,6 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR }) return res, nil, nil } - func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3) ([]*v3.Result, map[string]error, error) { // List query has support for only one query // we are skipping for PanelTypeTrace as it has a custom order by regardless of what's in the payload From a7fa0bb4e44227e4c36066568b301e0a991694f1 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 15 Nov 2024 16:33:10 +0530 Subject: [PATCH 26/41] fix: update pagination logic --- pkg/query-service/app/querier/querier.go | 82 +++-- pkg/query-service/app/querier/querier_test.go | 291 +++++++++++++++++ pkg/query-service/app/querier/v2/querier.go | 88 ++++-- .../app/querier/v2/querier_test.go | 295 +++++++++++++++++- 4 files changed, 704 insertions(+), 52 deletions(-) diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index d9d6e14129..6adbc032cd 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -12,7 +12,6 @@ import ( "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" - "go.signoz.io/signoz/pkg/query-service/common" chErrors "go.signoz.io/signoz/pkg/query-service/errors" "go.signoz.io/signoz/pkg/query-service/querycache" @@ -322,40 +321,49 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR qName := "" pageSize := uint64(0) limit := uint64(0) + offset := uint64(0) // se we are considering only one query for name, v := range params.CompositeQuery.BuilderQueries { qName = name pageSize = v.PageSize + + // for traces specifically limit = v.Limit + offset = v.Offset } data := []*v3.Row{} + tracesLimit := limit + offset + for _, v := range tsRanges { params.Start = v.Start params.End = v.End - params.CompositeQuery.BuilderQueries[qName].PageSize = pageSize - uint64(len(data)) - queries, err := q.builder.PrepareQueries(params) - if err != nil { - return nil, nil, err - } - + length := uint64(0) // this will to run only once - for name, query := range queries { - rowList, err := q.reader.GetListResultV3(ctx, query) + + // appending the filter to get the next set of data + if params.CompositeQuery.BuilderQueries[qName].DataSource == v3.DataSourceLogs { + params.CompositeQuery.BuilderQueries[qName].PageSize = pageSize - uint64(len(data)) + queries, err := q.builder.PrepareQueries(params) if err != nil { - errs := []error{err} - errQuriesByName := map[string]error{ - name: err, + return nil, nil, err + } + for name, query := range queries { + rowList, err := q.reader.GetListResultV3(ctx, query) + if err != nil { + errs := []error{err} + errQuriesByName := map[string]error{ + name: err, + } + return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) } - return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) + length += uint64(len(rowList)) + data = append(data, rowList...) } - data = append(data, rowList...) - } - if params.CompositeQuery.BuilderQueries[qName].DataSource == v3.DataSourceLogs { - if len(data) > 0 { + if length > 0 { params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ Key: v3.AttributeKey{ Key: "id", @@ -371,20 +379,48 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR break } } else { + // TRACE // we are updating the offset and limit based on the number of traces we have found in the current timerange // eg - // 1)offset = 0, limit = 100, tsRanges = [t1, t10], [t10, 20], [t20, t30] + // // if 100 traces are there in [t1, t10] then 100 will return immediately. // if 10 traces are there in [t1, t10] then we get 10, set offset to 0 and limit to 90, search in the next timerange of [t10, 20] + // if we don't find any trace in [t1, t10], then we search in [t10, 20] with offset=0, limit=100 + // // 2) offset = 50, limit = 100, tsRanges = [t1, t10], [t10, 20], [t20, t30] - // If we find 100 traces in [t1, t10] then we return immediately - // If we finds 50 in [t1, t10] then it will set offset = 0 and limit = 50 and search in the next timerange of [t10, 20] - // if we don't find any trace in [t1, t10], then we search in [t10, 20] with offset=50, limit=100 - if len(data) > 0 { - params.CompositeQuery.BuilderQueries[qName].Offset = 0 - params.CompositeQuery.BuilderQueries[qName].Limit = limit - uint64(len(data)) + // + // If we find 150 traces with limit=150 and offset=0 in [t1, t10] then we return immediately 100 traces + // If we find 50 in [t1, t10] with limit=150 and offset=0 then it will set limit = 100 and offset=0 and search in the next timerange of [t10, 20] + // if we don't find any trace in [t1, t10], then we search in [t10, 20] with limit=150 and offset=0 + params.CompositeQuery.BuilderQueries[qName].Offset = 0 + params.CompositeQuery.BuilderQueries[qName].Limit = tracesLimit + queries, err := q.builder.PrepareQueries(params) + if err != nil { + return nil, nil, err + } + for name, query := range queries { + rowList, err := q.reader.GetListResultV3(ctx, query) + if err != nil { + errs := []error{err} + errQuriesByName := map[string]error{ + name: err, + } + return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) + } + length += uint64(len(rowList)) + + // skip the traces unless offset is 0 + for _, row := range rowList { + if offset == 0 { + data = append(data, row) + } else { + offset-- + } + } } + tracesLimit = tracesLimit - length if uint64(len(data)) >= limit { break diff --git a/pkg/query-service/app/querier/querier_test.go b/pkg/query-service/app/querier/querier_test.go index 6a28c2c267..fcc70ee139 100644 --- a/pkg/query-service/app/querier/querier_test.go +++ b/pkg/query-service/app/querier/querier_test.go @@ -5,15 +5,21 @@ import ( "encoding/json" "fmt" "math" + "regexp" "strings" "testing" "time" + cmock "github.com/srikanthccv/ClickHouse-go-mock" + "github.com/stretchr/testify/require" + "go.signoz.io/signoz/pkg/query-service/app/clickhouseReader" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" "go.signoz.io/signoz/pkg/query-service/cache/inmemory" + "go.signoz.io/signoz/pkg/query-service/featureManager" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/querycache" + "go.signoz.io/signoz/pkg/query-service/utils" ) func minTimestamp(series []*v3.Series) int64 { @@ -1124,3 +1130,288 @@ func TestQueryRangeValueTypePromQL(t *testing.T) { } } } + +type regexMatcher struct { +} + +func (m *regexMatcher) Match(expectedSQL, actualSQL string) error { + re, err := regexp.Compile(expectedSQL) + if err != nil { + return err + } + if !re.MatchString(actualSQL) { + return fmt.Errorf("expected query to contain %s, got %s", expectedSQL, actualSQL) + } + return nil +} + +func Test_querier_runWindowBasedListQuery(t *testing.T) { + params := &v3.QueryRangeParamsV3{ + Start: 1722171576000000000, // July 28, 2024 6:29:36 PM + End: 1722262800000000000, // July 29, 2024 7:50:00 PM + CompositeQuery: &v3.CompositeQuery{ + PanelType: v3.PanelTypeList, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + Expression: "A", + DataSource: v3.DataSourceTraces, + PageSize: 10, + Limit: 100, + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorNoOp, + SelectColumns: []v3.AttributeKey{{Key: "serviceName"}}, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{}, + }, + }, + }, + }, + } + + tsRanges := []utils.LogsListTsRange{ + { + Start: 1722259200000000000, // July 29, 2024 6:50:00 PM + End: 1722262800000000000, // July 29, 2024 7:50:00 PM + }, + { + Start: 1722252000000000000, // July 29, 2024 4:50:00 PM + End: 1722259200000000000, // July 29, 2024 6:50:00 PM + }, + { + Start: 1722237600000000000, // July 29, 2024 12:50:00 PM + End: 1722252000000000000, // July 29, 2024 4:50:00 PM + }, + { + Start: 1722208800000000000, // July 29, 2024 4:50:00 AM + End: 1722237600000000000, // July 29, 2024 12:50:00 PM + }, + { + Start: 1722171576000000000, // July 28, 2024 6:29:36 PM + End: 1722208800000000000, // July 29, 2024 4:50:00 AM + }, + } + + type queryParams struct { + start int64 + end int64 + limit uint64 + offset uint64 + } + + type queryResponse struct { + expectedQuery string + timestamps []uint64 + } + + // create test struct with moc data i.e array of timestamps, limit, offset and expected results + testCases := []struct { + name string + queryResponses []queryResponse + queryParams queryParams + expectedTimestamps []int64 + }{ + { + name: "should return correct timestamps when querying within time window", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 2", + timestamps: []uint64{1722259300000000000, 1722259400000000000}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 2, + offset: 0, + }, + expectedTimestamps: []int64{1722259300000000000, 1722259400000000000}, + }, + { + name: "all data not in first windows", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 3", + timestamps: []uint64{1722259300000000000, 1722259400000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722252000000000000' AND timestamp <= '1722259200000000000').* DESC LIMIT 1", + timestamps: []uint64{1722253000000000000}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 3, + offset: 0, + }, + expectedTimestamps: []int64{1722259300000000000, 1722259400000000000, 1722253000000000000}, + }, + { + name: "data in multiple windows", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 5", + timestamps: []uint64{1722259300000000000, 1722259400000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722252000000000000' AND timestamp <= '1722259200000000000').* DESC LIMIT 3", + timestamps: []uint64{1722253000000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722237600000000000' AND timestamp <= '1722252000000000000').* DESC LIMIT 2", + timestamps: []uint64{1722237700000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722208800000000000' AND timestamp <= '1722237600000000000').* DESC LIMIT 1", + timestamps: []uint64{}, + }, + { + expectedQuery: ".*(timestamp >= '1722171576000000000' AND timestamp <= '1722208800000000000').* DESC LIMIT 1", + timestamps: []uint64{}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 5, + offset: 0, + }, + expectedTimestamps: []int64{1722259300000000000, 1722259400000000000, 1722253000000000000, 1722237700000000000}, + }, + { + name: "query with offset", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 7", + timestamps: []uint64{1722259210000000000, 1722259220000000000, 1722259230000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722252000000000000' AND timestamp <= '1722259200000000000').* DESC LIMIT 4", + timestamps: []uint64{1722253000000000000, 1722254000000000000, 1722255000000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722237600000000000' AND timestamp <= '1722252000000000000').* DESC LIMIT 1", + timestamps: []uint64{1722237700000000000}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 4, + offset: 3, + }, + expectedTimestamps: []int64{1722253000000000000, 1722254000000000000, 1722255000000000000, 1722237700000000000}, + }, + { + name: "query with offset and limit- data spread across multiple windows", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 11", + timestamps: []uint64{}, + }, + { + expectedQuery: ".*(timestamp >= '1722252000000000000' AND timestamp <= '1722259200000000000').* DESC LIMIT 11", + timestamps: []uint64{1722253000000000000, 1722254000000000000, 1722255000000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722237600000000000' AND timestamp <= '1722252000000000000').* DESC LIMIT 8", + timestamps: []uint64{1722237700000000000, 1722237800000000000, 1722237900000000000, 1722237910000000000, 1722237920000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722208800000000000' AND timestamp <= '1722237600000000000').* DESC LIMIT 3", + timestamps: []uint64{1722208810000000000, 1722208820000000000, 1722208830000000000}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 5, + offset: 6, + }, + expectedTimestamps: []int64{1722237910000000000, 1722237920000000000, 1722208810000000000, 1722208820000000000, 1722208830000000000}, + }, + } + + cols := []cmock.ColumnType{ + {Name: "timestamp", Type: "UInt64"}, + {Name: "name", Type: "String"}, + } + testName := "name" + + options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") + + // iterate over test data, create reader and run test + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Setup mock + mock, err := cmock.NewClickHouseWithQueryMatcher(nil, ®exMatcher{}) + require.NoError(t, err, "Failed to create ClickHouse mock") + + // Configure mock responses + for _, response := range tc.queryResponses { + values := make([][]any, 0, len(response.timestamps)) + for _, ts := range response.timestamps { + values = append(values, []any{&ts, &testName}) + } + // if len(values) > 0 { + mock.ExpectQuery(response.expectedQuery).WillReturnRows( + cmock.NewRows(cols, values), + ) + // } + } + + // Create reader and querier + reader := clickhouseReader.NewReaderFromClickhouseConnection( + mock, + options, + nil, + "", + featureManager.StartManager(), + "", + true, + true, + ) + + q := &querier{ + reader: reader, + builder: queryBuilder.NewQueryBuilder( + queryBuilder.QueryBuilderOptions{ + BuildTraceQuery: tracesV3.PrepareTracesQuery, + }, + featureManager.StartManager(), + ), + } + // Update query parameters + params.Start = tc.queryParams.start + params.End = tc.queryParams.end + params.CompositeQuery.BuilderQueries["A"].Limit = tc.queryParams.limit + params.CompositeQuery.BuilderQueries["A"].Offset = tc.queryParams.offset + + // Execute query + results, errMap, err := q.runWindowBasedListQuery(context.Background(), params, tsRanges) + + // Assertions + require.NoError(t, err, "Query execution failed") + require.Nil(t, errMap, "Unexpected error map in results") + require.Len(t, results, 1, "Expected exactly one result set") + + result := results[0] + require.Equal(t, "A", result.QueryName, "Incorrect query name in results") + require.Len(t, result.List, len(tc.expectedTimestamps), + "Result count mismatch: got %d results, expected %d", + len(result.List), len(tc.expectedTimestamps)) + + for i, expected := range tc.expectedTimestamps { + require.Equal(t, expected, result.List[i].Timestamp.UnixNano(), + "Timestamp mismatch at index %d: got %d, expected %d", + i, result.List[i].Timestamp.UnixNano(), expected) + } + + // Verify mock expectations + err = mock.ExpectationsWereMet() + require.NoError(t, err, "Mock expectations were not met") + }) + } +} diff --git a/pkg/query-service/app/querier/v2/querier.go b/pkg/query-service/app/querier/v2/querier.go index 06c96a4a0e..11944029c6 100644 --- a/pkg/query-service/app/querier/v2/querier.go +++ b/pkg/query-service/app/querier/v2/querier.go @@ -322,40 +322,49 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR qName := "" pageSize := uint64(0) limit := uint64(0) + offset := uint64(0) // se we are considering only one query for name, v := range params.CompositeQuery.BuilderQueries { qName = name pageSize = v.PageSize + + // for traces specifically limit = v.Limit + offset = v.Offset } data := []*v3.Row{} + tracesLimit := limit + offset + for _, v := range tsRanges { params.Start = v.Start params.End = v.End - params.CompositeQuery.BuilderQueries[qName].PageSize = pageSize - uint64(len(data)) - queries, err := q.builder.PrepareQueries(params) - if err != nil { - return nil, nil, err - } - + length := uint64(0) // this will to run only once - for name, query := range queries { - rowList, err := q.reader.GetListResultV3(ctx, query) + + // appending the filter to get the next set of data + if params.CompositeQuery.BuilderQueries[qName].DataSource == v3.DataSourceLogs { + params.CompositeQuery.BuilderQueries[qName].PageSize = pageSize - uint64(len(data)) + queries, err := q.builder.PrepareQueries(params) if err != nil { - errs := []error{err} - errQuriesByName := map[string]error{ - name: err, + return nil, nil, err + } + for name, query := range queries { + rowList, err := q.reader.GetListResultV3(ctx, query) + if err != nil { + errs := []error{err} + errQuriesByName := map[string]error{ + name: err, + } + return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) } - return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) + length += uint64(len(rowList)) + data = append(data, rowList...) } - data = append(data, rowList...) - } - if params.CompositeQuery.BuilderQueries[qName].DataSource == v3.DataSourceLogs { - if len(data) > 0 { + if length > 0 { params.CompositeQuery.BuilderQueries[qName].Filters.Items = append(params.CompositeQuery.BuilderQueries[qName].Filters.Items, v3.FilterItem{ Key: v3.AttributeKey{ Key: "id", @@ -371,29 +380,53 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR break } } else { + // TRACE // we are updating the offset and limit based on the number of traces we have found in the current timerange // eg - // 1)offset = 0, limit = 100, tsRanges = [t1, t10], [t10, 20], [t20, t30] + // // if 100 traces are there in [t1, t10] then 100 will return immediately. // if 10 traces are there in [t1, t10] then we get 10, set offset to 0 and limit to 90, search in the next timerange of [t10, 20] + // if we don't find any trace in [t1, t10], then we search in [t10, 20] with offset=0, limit=100 + // // 2) offset = 50, limit = 100, tsRanges = [t1, t10], [t10, 20], [t20, t30] - // If we find 100 traces in [t1, t10] then we return immediately - // If we finds 50 in [t1, t10] then it will set offset = 0 and limit = 50 and search in the next timerange of [t10, 20] - // if we don't find any trace in [t1, t10], then we search in [t10, 20] with offset=50, limit=100 - if len(data) > 0 { - params.CompositeQuery.BuilderQueries[qName].Offset = 0 - params.CompositeQuery.BuilderQueries[qName].Limit = limit - uint64(len(data)) + // + // If we find 150 traces with limit=150 and offset=0 in [t1, t10] then we return immediately 100 traces + // If we find 50 in [t1, t10] with limit=150 and offset=0 then it will set limit = 100 and offset=0 and search in the next timerange of [t10, 20] + // if we don't find any trace in [t1, t10], then we search in [t10, 20] with limit=150 and offset=0 + params.CompositeQuery.BuilderQueries[qName].Offset = 0 + params.CompositeQuery.BuilderQueries[qName].Limit = tracesLimit + queries, err := q.builder.PrepareQueries(params) + if err != nil { + return nil, nil, err } + for name, query := range queries { + rowList, err := q.reader.GetListResultV3(ctx, query) + if err != nil { + errs := []error{err} + errQuriesByName := map[string]error{ + name: err, + } + return nil, errQuriesByName, fmt.Errorf("encountered multiple errors: %s", multierr.Combine(errs...)) + } + length += uint64(len(rowList)) + + // skip the traces unless offset is 0 + for _, row := range rowList { + if offset == 0 { + data = append(data, row) + } else { + offset-- + } + } + } + tracesLimit = tracesLimit - length if uint64(len(data)) >= limit { break } } - - if uint64(len(data)) >= pageSize { - break - } } res = append(res, &v3.Result{ QueryName: qName, @@ -401,8 +434,9 @@ func (q *querier) runWindowBasedListQuery(ctx context.Context, params *v3.QueryR }) return res, nil, nil } + func (q *querier) runBuilderListQueries(ctx context.Context, params *v3.QueryRangeParamsV3) ([]*v3.Result, map[string]error, error) { - // List query has support for only one query + // List query has support for only one query. // we are skipping for PanelTypeTrace as it has a custom order by regardless of what's in the payload if params.CompositeQuery != nil && len(params.CompositeQuery.BuilderQueries) == 1 && diff --git a/pkg/query-service/app/querier/v2/querier_test.go b/pkg/query-service/app/querier/v2/querier_test.go index 62e18d244d..f55530a420 100644 --- a/pkg/query-service/app/querier/v2/querier_test.go +++ b/pkg/query-service/app/querier/v2/querier_test.go @@ -5,15 +5,21 @@ import ( "encoding/json" "fmt" "math" + "regexp" "strings" "testing" "time" + cmock "github.com/srikanthccv/ClickHouse-go-mock" + "github.com/stretchr/testify/require" + "go.signoz.io/signoz/pkg/query-service/app/clickhouseReader" "go.signoz.io/signoz/pkg/query-service/app/queryBuilder" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" "go.signoz.io/signoz/pkg/query-service/cache/inmemory" + "go.signoz.io/signoz/pkg/query-service/featureManager" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/querycache" + "go.signoz.io/signoz/pkg/query-service/utils" ) func minTimestamp(series []*v3.Series) int64 { @@ -798,8 +804,8 @@ func TestV2QueryRangeValueType(t *testing.T) { } q := NewQuerier(opts) expectedTimeRangeInQueryString := []string{ - fmt.Sprintf("unix_milli >= %d AND unix_milli < %d", 1675115520000, 1675115580000+120*60*1000), // 31st Jan, 03:23:00 to 31st Jan, 05:23:00 - fmt.Sprintf("unix_milli >= %d AND unix_milli < %d", 1675115580000+120*60*1000, 1675115580000+180*60*1000), // 31st Jan, 05:23:00 to 31st Jan, 06:23:00 + fmt.Sprintf("unix_milli >= %d AND unix_milli < %d", 1675115520000, 1675115580000+120*60*1000), // 31st Jan, 03:23:00 to 31st Jan, 05:23:00 + fmt.Sprintf("unix_milli >= %d AND unix_milli < %d", 1675115580000+120*60*1000, 1675115580000+180*60*1000), // 31st Jan, 05:23:00 to 31st Jan, 06:23:00 fmt.Sprintf("timestamp >= '%d' AND timestamp <= '%d'", (1675119196722)*int64(1000000), (1675126396722)*int64(1000000)), // 31st Jan, 05:23:00 to 31st Jan, 06:23:00 } @@ -1178,3 +1184,288 @@ func TestV2QueryRangeValueTypePromQL(t *testing.T) { } } } + +type regexMatcher struct { +} + +func (m *regexMatcher) Match(expectedSQL, actualSQL string) error { + re, err := regexp.Compile(expectedSQL) + if err != nil { + return err + } + if !re.MatchString(actualSQL) { + return fmt.Errorf("expected query to contain %s, got %s", expectedSQL, actualSQL) + } + return nil +} + +func Test_querier_runWindowBasedListQuery(t *testing.T) { + params := &v3.QueryRangeParamsV3{ + Start: 1722171576000000000, // July 28, 2024 6:29:36 PM + End: 1722262800000000000, // July 29, 2024 7:50:00 PM + CompositeQuery: &v3.CompositeQuery{ + PanelType: v3.PanelTypeList, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + Expression: "A", + DataSource: v3.DataSourceTraces, + PageSize: 10, + Limit: 100, + StepInterval: 60, + AggregateOperator: v3.AggregateOperatorNoOp, + SelectColumns: []v3.AttributeKey{{Key: "serviceName"}}, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{}, + }, + }, + }, + }, + } + + tsRanges := []utils.LogsListTsRange{ + { + Start: 1722259200000000000, // July 29, 2024 6:50:00 PM + End: 1722262800000000000, // July 29, 2024 7:50:00 PM + }, + { + Start: 1722252000000000000, // July 29, 2024 4:50:00 PM + End: 1722259200000000000, // July 29, 2024 6:50:00 PM + }, + { + Start: 1722237600000000000, // July 29, 2024 12:50:00 PM + End: 1722252000000000000, // July 29, 2024 4:50:00 PM + }, + { + Start: 1722208800000000000, // July 29, 2024 4:50:00 AM + End: 1722237600000000000, // July 29, 2024 12:50:00 PM + }, + { + Start: 1722171576000000000, // July 28, 2024 6:29:36 PM + End: 1722208800000000000, // July 29, 2024 4:50:00 AM + }, + } + + type queryParams struct { + start int64 + end int64 + limit uint64 + offset uint64 + } + + type queryResponse struct { + expectedQuery string + timestamps []uint64 + } + + // create test struct with moc data i.e array of timestamps, limit, offset and expected results + testCases := []struct { + name string + queryResponses []queryResponse + queryParams queryParams + expectedTimestamps []int64 + }{ + { + name: "should return correct timestamps when querying within time window", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 2", + timestamps: []uint64{1722259300000000000, 1722259400000000000}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 2, + offset: 0, + }, + expectedTimestamps: []int64{1722259300000000000, 1722259400000000000}, + }, + { + name: "all data not in first windows", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 3", + timestamps: []uint64{1722259300000000000, 1722259400000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722252000000000000' AND timestamp <= '1722259200000000000').* DESC LIMIT 1", + timestamps: []uint64{1722253000000000000}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 3, + offset: 0, + }, + expectedTimestamps: []int64{1722259300000000000, 1722259400000000000, 1722253000000000000}, + }, + { + name: "data in multiple windows", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 5", + timestamps: []uint64{1722259300000000000, 1722259400000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722252000000000000' AND timestamp <= '1722259200000000000').* DESC LIMIT 3", + timestamps: []uint64{1722253000000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722237600000000000' AND timestamp <= '1722252000000000000').* DESC LIMIT 2", + timestamps: []uint64{1722237700000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722208800000000000' AND timestamp <= '1722237600000000000').* DESC LIMIT 1", + timestamps: []uint64{}, + }, + { + expectedQuery: ".*(timestamp >= '1722171576000000000' AND timestamp <= '1722208800000000000').* DESC LIMIT 1", + timestamps: []uint64{}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 5, + offset: 0, + }, + expectedTimestamps: []int64{1722259300000000000, 1722259400000000000, 1722253000000000000, 1722237700000000000}, + }, + { + name: "query with offset", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 7", + timestamps: []uint64{1722259210000000000, 1722259220000000000, 1722259230000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722252000000000000' AND timestamp <= '1722259200000000000').* DESC LIMIT 4", + timestamps: []uint64{1722253000000000000, 1722254000000000000, 1722255000000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722237600000000000' AND timestamp <= '1722252000000000000').* DESC LIMIT 1", + timestamps: []uint64{1722237700000000000}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 4, + offset: 3, + }, + expectedTimestamps: []int64{1722253000000000000, 1722254000000000000, 1722255000000000000, 1722237700000000000}, + }, + { + name: "query with offset and limit- data spread across multiple windows", + queryResponses: []queryResponse{ + { + expectedQuery: ".*(timestamp >= '1722259200000000000' AND timestamp <= '1722262800000000000').* DESC LIMIT 11", + timestamps: []uint64{}, + }, + { + expectedQuery: ".*(timestamp >= '1722252000000000000' AND timestamp <= '1722259200000000000').* DESC LIMIT 11", + timestamps: []uint64{1722253000000000000, 1722254000000000000, 1722255000000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722237600000000000' AND timestamp <= '1722252000000000000').* DESC LIMIT 8", + timestamps: []uint64{1722237700000000000, 1722237800000000000, 1722237900000000000, 1722237910000000000, 1722237920000000000}, + }, + { + expectedQuery: ".*(timestamp >= '1722208800000000000' AND timestamp <= '1722237600000000000').* DESC LIMIT 3", + timestamps: []uint64{1722208810000000000, 1722208820000000000, 1722208830000000000}, + }, + }, + queryParams: queryParams{ + start: 1722171576000000000, + end: 1722262800000000000, + limit: 5, + offset: 6, + }, + expectedTimestamps: []int64{1722237910000000000, 1722237920000000000, 1722208810000000000, 1722208820000000000, 1722208830000000000}, + }, + } + + cols := []cmock.ColumnType{ + {Name: "timestamp", Type: "UInt64"}, + {Name: "name", Type: "String"}, + } + testName := "name" + + options := clickhouseReader.NewOptions("", 0, 0, 0, "", "archiveNamespace") + + // iterate over test data, create reader and run test + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Setup mock + mock, err := cmock.NewClickHouseWithQueryMatcher(nil, ®exMatcher{}) + require.NoError(t, err, "Failed to create ClickHouse mock") + + // Configure mock responses + for _, response := range tc.queryResponses { + values := make([][]any, 0, len(response.timestamps)) + for _, ts := range response.timestamps { + values = append(values, []any{&ts, &testName}) + } + // if len(values) > 0 { + mock.ExpectQuery(response.expectedQuery).WillReturnRows( + cmock.NewRows(cols, values), + ) + // } + } + + // Create reader and querier + reader := clickhouseReader.NewReaderFromClickhouseConnection( + mock, + options, + nil, + "", + featureManager.StartManager(), + "", + true, + true, + ) + + q := &querier{ + reader: reader, + builder: queryBuilder.NewQueryBuilder( + queryBuilder.QueryBuilderOptions{ + BuildTraceQuery: tracesV3.PrepareTracesQuery, + }, + featureManager.StartManager(), + ), + } + // Update query parameters + params.Start = tc.queryParams.start + params.End = tc.queryParams.end + params.CompositeQuery.BuilderQueries["A"].Limit = tc.queryParams.limit + params.CompositeQuery.BuilderQueries["A"].Offset = tc.queryParams.offset + + // Execute query + results, errMap, err := q.runWindowBasedListQuery(context.Background(), params, tsRanges) + + // Assertions + require.NoError(t, err, "Query execution failed") + require.Nil(t, errMap, "Unexpected error map in results") + require.Len(t, results, 1, "Expected exactly one result set") + + result := results[0] + require.Equal(t, "A", result.QueryName, "Incorrect query name in results") + require.Len(t, result.List, len(tc.expectedTimestamps), + "Result count mismatch: got %d results, expected %d", + len(result.List), len(tc.expectedTimestamps)) + + for i, expected := range tc.expectedTimestamps { + require.Equal(t, expected, result.List[i].Timestamp.UnixNano(), + "Timestamp mismatch at index %d: got %d, expected %d", + i, result.List[i].Timestamp.UnixNano(), expected) + } + + // Verify mock expectations + err = mock.ExpectationsWereMet() + require.NoError(t, err, "Mock expectations were not met") + }) + } +} From 50bc68a4dc6ae077f1566696052fd85c0bb4daaf Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Sun, 17 Nov 2024 10:54:38 +0530 Subject: [PATCH 27/41] fix: remove utils --- pkg/query-service/app/traces/v4/utils.go | 216 ----------------------- 1 file changed, 216 deletions(-) delete mode 100644 pkg/query-service/app/traces/v4/utils.go diff --git a/pkg/query-service/app/traces/v4/utils.go b/pkg/query-service/app/traces/v4/utils.go deleted file mode 100644 index ae132f6d9e..0000000000 --- a/pkg/query-service/app/traces/v4/utils.go +++ /dev/null @@ -1,216 +0,0 @@ -package v4 - -import ( - "strconv" - - v3 "go.signoz.io/signoz/pkg/query-service/model/v3" - "go.signoz.io/signoz/pkg/query-service/utils" - "go.uber.org/zap" -) - -var TracesListViewDefaultSelectedColumns = []v3.AttributeKey{ - { - Key: "serviceName", - DataType: v3.AttributeKeyDataTypeString, - Type: v3.AttributeKeyTypeTag, - IsColumn: true, - }, - { - Key: "name", - DataType: v3.AttributeKeyDataTypeString, - Type: v3.AttributeKeyTypeTag, - IsColumn: true, - }, - { - Key: "durationNano", - DataType: v3.AttributeKeyDataTypeArrayFloat64, - Type: v3.AttributeKeyTypeTag, - IsColumn: true, - }, - { - Key: "httpMethod", - DataType: v3.AttributeKeyDataTypeString, - Type: v3.AttributeKeyTypeTag, - IsColumn: true, - }, - { - Key: "responseStatusCode", - DataType: v3.AttributeKeyDataTypeString, - Type: v3.AttributeKeyTypeTag, - IsColumn: true, - }, -} - -// check if traceId filter is used in traces query and return the list of traceIds -func TraceIdFilterUsedWithEqual(params *v3.QueryRangeParamsV3) (bool, []string) { - compositeQuery := params.CompositeQuery - if compositeQuery == nil { - return false, []string{} - } - var traceIds []string - var traceIdFilterUsed bool - - // Build queries for each builder query - for queryName, query := range compositeQuery.BuilderQueries { - if query.Expression != queryName && query.DataSource != v3.DataSourceTraces { - continue - } - - // check filter attribute - if query.Filters != nil && len(query.Filters.Items) != 0 { - for _, item := range query.Filters.Items { - - if item.Key.Key == "traceID" && (item.Operator == v3.FilterOperatorIn || - item.Operator == v3.FilterOperatorEqual) { - traceIdFilterUsed = true - // validate value - var err error - val := item.Value - val, err = utils.ValidateAndCastValue(val, item.Key.DataType) - if err != nil { - zap.L().Error("invalid value for key", zap.String("key", item.Key.Key), zap.Error(err)) - return false, []string{} - } - if val != nil { - fmtVal := extractFormattedStringValues(val) - traceIds = append(traceIds, fmtVal...) - } - } - } - } - - } - - zap.L().Debug("traceIds", zap.Any("traceIds", traceIds)) - return traceIdFilterUsed, traceIds -} - -func extractFormattedStringValues(v interface{}) []string { - // if it's pointer convert it to a value - v = getPointerValue(v) - - switch x := v.(type) { - case string: - return []string{x} - - case []interface{}: - if len(x) == 0 { - return []string{} - } - switch x[0].(type) { - case string: - values := []string{} - for _, val := range x { - values = append(values, val.(string)) - } - return values - default: - return []string{} - } - default: - return []string{} - } -} - -func getPointerValue(v interface{}) interface{} { - switch x := v.(type) { - case *uint8: - return *x - case *uint16: - return *x - case *uint32: - return *x - case *uint64: - return *x - case *int: - return *x - case *int8: - return *x - case *int16: - return *x - case *int32: - return *x - case *int64: - return *x - case *float32: - return *x - case *float64: - return *x - case *string: - return *x - case *bool: - return *x - case []interface{}: - values := []interface{}{} - for _, val := range x { - values = append(values, getPointerValue(val)) - } - return values - default: - return v - } -} - -func AddTimestampFilters(minTime int64, maxTime int64, params *v3.QueryRangeParamsV3) { - if minTime == 0 && maxTime == 0 { - return - } - - compositeQuery := params.CompositeQuery - if compositeQuery == nil { - return - } - // Build queries for each builder query and apply timestamp filter only if TraceID is present - for queryName, query := range compositeQuery.BuilderQueries { - if query.Expression != queryName && query.DataSource != v3.DataSourceTraces { - continue - } - - addTimeStampFilter := false - - // check filter attribute - if query.Filters != nil && len(query.Filters.Items) != 0 { - for _, item := range query.Filters.Items { - if item.Key.Key == "traceID" && (item.Operator == v3.FilterOperatorIn || - item.Operator == v3.FilterOperatorEqual) { - addTimeStampFilter = true - } - } - } - - // add timestamp filter to query only if traceID filter along with equal/similar operator is used - if addTimeStampFilter { - timeFilters := []v3.FilterItem{ - { - Key: v3.AttributeKey{ - Key: "timestamp", - Type: v3.AttributeKeyTypeTag, - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, - Value: strconv.FormatUint(uint64(minTime), 10), - Operator: v3.FilterOperatorGreaterThanOrEq, - }, - { - Key: v3.AttributeKey{ - Key: "timestamp", - Type: v3.AttributeKeyTypeTag, - DataType: v3.AttributeKeyDataTypeString, - IsColumn: true, - }, - Value: strconv.FormatUint(uint64(maxTime), 10), - Operator: v3.FilterOperatorLessThanOrEq, - }, - } - - // add new timestamp filter to query - if query.Filters == nil { - query.Filters = &v3.FilterSet{ - Items: timeFilters, - } - } else { - query.Filters.Items = append(query.Filters.Items, timeFilters...) - } - } - } -} From d817df23f873565ef3ba156038022ee611b8d68a Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Sun, 17 Nov 2024 11:23:55 +0530 Subject: [PATCH 28/41] fix: remove unwanted API's --- .../app/clickhouseReader/reader.go | 798 +----------------- pkg/query-service/app/http_handler.go | 86 -- pkg/query-service/interfaces/interface.go | 5 - 3 files changed, 6 insertions(+), 883 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 7c2946d383..fbd53da329 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -485,8 +485,13 @@ func (r *ClickHouseReader) GetQueryRangeResult(ctx context.Context, query *model } func (r *ClickHouseReader) GetServicesList(ctx context.Context) (*[]string, error) { + services := []string{} - query := fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE ts_bucket_start > (toUnixTimestamp(now()) - 1800) AND toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceLocalTableName) + query := fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceLocalTableName) + + if r.useTraceNewSchema { + query = fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE ts_bucket_start > (toUnixTimestamp(now()) - 1800) AND toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceLocalTableName) + } rows, err := r.db.Query(ctx, query) @@ -798,307 +803,6 @@ func buildFilterArrayQuery(_ context.Context, excludeMap map[string]struct{}, pa return args } -func (r *ClickHouseReader) GetSpanFilters(ctx context.Context, queryParams *model.SpanFilterParams) (*model.SpanFiltersResponse, *model.ApiError) { - - var query string - excludeMap := make(map[string]struct{}) - for _, e := range queryParams.Exclude { - if e == constants.OperationRequest { - excludeMap[constants.OperationDB] = struct{}{} - continue - } - excludeMap[e] = struct{}{} - } - - args := []interface{}{clickhouse.Named("timestampL", strconv.FormatInt(queryParams.Start.UnixNano(), 10)), clickhouse.Named("timestampU", strconv.FormatInt(queryParams.End.UnixNano(), 10))} - if len(queryParams.TraceID) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.TraceID, constants.TraceID, &query, args) - } - if len(queryParams.ServiceName) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.ServiceName, constants.ServiceName, &query, args) - } - if len(queryParams.HttpRoute) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpRoute, constants.HttpRoute, &query, args) - } - if len(queryParams.HttpHost) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpHost, constants.HttpHost, &query, args) - } - if len(queryParams.HttpMethod) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpMethod, constants.HttpMethod, &query, args) - } - if len(queryParams.HttpUrl) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpUrl, constants.HttpUrl, &query, args) - } - if len(queryParams.Operation) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.Operation, constants.OperationDB, &query, args) - } - if len(queryParams.RPCMethod) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.RPCMethod, constants.RPCMethod, &query, args) - } - if len(queryParams.ResponseStatusCode) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.ResponseStatusCode, constants.ResponseStatusCode, &query, args) - } - - if len(queryParams.MinDuration) != 0 { - query = query + " AND durationNano >= @durationNanoMin" - args = append(args, clickhouse.Named("durationNanoMin", queryParams.MinDuration)) - } - if len(queryParams.MaxDuration) != 0 { - query = query + " AND durationNano <= @durationNanoMax" - args = append(args, clickhouse.Named("durationNanoMax", queryParams.MaxDuration)) - } - - if len(queryParams.SpanKind) != 0 { - query = query + " AND kind = @kind" - args = append(args, clickhouse.Named("kind", queryParams.SpanKind)) - } - - query = getStatusFilters(query, queryParams.Status, excludeMap) - - traceFilterReponse := model.SpanFiltersResponse{ - Status: map[string]uint64{}, - Duration: map[string]uint64{}, - ServiceName: map[string]uint64{}, - Operation: map[string]uint64{}, - ResponseStatusCode: map[string]uint64{}, - RPCMethod: map[string]uint64{}, - HttpMethod: map[string]uint64{}, - HttpUrl: map[string]uint64{}, - HttpRoute: map[string]uint64{}, - HttpHost: map[string]uint64{}, - } - - for _, e := range queryParams.GetFilters { - switch e { - case constants.TraceID: - continue - case constants.ServiceName: - finalQuery := fmt.Sprintf("SELECT serviceName, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) - finalQuery += query - finalQuery += " GROUP BY serviceName" - var dBResponse []model.DBResponseServiceName - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - for _, service := range dBResponse { - if service.ServiceName != "" { - traceFilterReponse.ServiceName[service.ServiceName] = service.Count - } - } - case constants.HttpRoute: - finalQuery := fmt.Sprintf("SELECT httpRoute, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) - finalQuery += query - finalQuery += " GROUP BY httpRoute" - var dBResponse []model.DBResponseHttpRoute - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - for _, service := range dBResponse { - if service.HttpRoute != "" { - traceFilterReponse.HttpRoute[service.HttpRoute] = service.Count - } - } - case constants.HttpUrl: - finalQuery := fmt.Sprintf("SELECT httpUrl, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) - finalQuery += query - finalQuery += " GROUP BY httpUrl" - var dBResponse []model.DBResponseHttpUrl - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - for _, service := range dBResponse { - if service.HttpUrl != "" { - traceFilterReponse.HttpUrl[service.HttpUrl] = service.Count - } - } - case constants.HttpMethod: - finalQuery := fmt.Sprintf("SELECT httpMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) - finalQuery += query - finalQuery += " GROUP BY httpMethod" - var dBResponse []model.DBResponseHttpMethod - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - for _, service := range dBResponse { - if service.HttpMethod != "" { - traceFilterReponse.HttpMethod[service.HttpMethod] = service.Count - } - } - case constants.HttpHost: - finalQuery := fmt.Sprintf("SELECT httpHost, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) - finalQuery += query - finalQuery += " GROUP BY httpHost" - var dBResponse []model.DBResponseHttpHost - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - for _, service := range dBResponse { - if service.HttpHost != "" { - traceFilterReponse.HttpHost[service.HttpHost] = service.Count - } - } - case constants.OperationRequest: - finalQuery := fmt.Sprintf("SELECT name, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) - finalQuery += query - finalQuery += " GROUP BY name" - var dBResponse []model.DBResponseOperation - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - for _, service := range dBResponse { - if service.Operation != "" { - traceFilterReponse.Operation[service.Operation] = service.Count - } - } - case constants.Status: - finalQuery := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = true", r.TraceDB, r.traceTableName) - finalQuery += query - var dBResponse []model.DBResponseTotal - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - - finalQuery2 := fmt.Sprintf("SELECT COUNT(*) as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU AND hasError = false", r.TraceDB, r.traceTableName) - finalQuery2 += query - var dBResponse2 []model.DBResponseTotal - err = r.db.Select(ctx, &dBResponse2, finalQuery2, args...) - zap.L().Info(finalQuery2) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - if len(dBResponse) > 0 && len(dBResponse2) > 0 { - traceFilterReponse.Status = map[string]uint64{"ok": dBResponse2[0].NumTotal, "error": dBResponse[0].NumTotal} - } else if len(dBResponse) > 0 { - traceFilterReponse.Status = map[string]uint64{"ok": 0, "error": dBResponse[0].NumTotal} - } else if len(dBResponse2) > 0 { - traceFilterReponse.Status = map[string]uint64{"ok": dBResponse2[0].NumTotal, "error": 0} - } else { - traceFilterReponse.Status = map[string]uint64{"ok": 0, "error": 0} - } - case constants.Duration: - err := r.featureFlags.CheckFeature(constants.DurationSort) - durationSortEnabled := err == nil - finalQuery := "" - if !durationSortEnabled { - // if duration sort is not enabled, we need to get the min and max duration from the index table - finalQuery = fmt.Sprintf("SELECT min(durationNano) as min, max(durationNano) as max FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) - finalQuery += query - var dBResponse []model.DBResponseMinMax - err = r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - if len(dBResponse) > 0 { - traceFilterReponse.Duration = map[string]uint64{"minDuration": dBResponse[0].Min, "maxDuration": dBResponse[0].Max} - } - } else { - // when duration sort is enabled, we need to get the min and max duration from the duration table - finalQuery = fmt.Sprintf("SELECT durationNano as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.durationTable) - finalQuery += query - finalQuery += " ORDER BY durationNano LIMIT 1" - var dBResponse []model.DBResponseTotal - err = r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - - finalQuery = fmt.Sprintf("SELECT durationNano as numTotal FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.durationTable) - finalQuery += query - finalQuery += " ORDER BY durationNano DESC LIMIT 1" - var dBResponse2 []model.DBResponseTotal - err = r.db.Select(ctx, &dBResponse2, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - if len(dBResponse) > 0 { - traceFilterReponse.Duration["minDuration"] = dBResponse[0].NumTotal - } - if len(dBResponse2) > 0 { - traceFilterReponse.Duration["maxDuration"] = dBResponse2[0].NumTotal - } - } - case constants.RPCMethod: - finalQuery := fmt.Sprintf("SELECT rpcMethod, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) - finalQuery += query - finalQuery += " GROUP BY rpcMethod" - var dBResponse []model.DBResponseRPCMethod - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - for _, service := range dBResponse { - if service.RPCMethod != "" { - traceFilterReponse.RPCMethod[service.RPCMethod] = service.Count - } - } - - case constants.ResponseStatusCode: - finalQuery := fmt.Sprintf("SELECT responseStatusCode, count() as count FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", r.TraceDB, r.traceTableName) - finalQuery += query - finalQuery += " GROUP BY responseStatusCode" - var dBResponse []model.DBResponseStatusCodeMethod - err := r.db.Select(ctx, &dBResponse, finalQuery, args...) - zap.L().Info(finalQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query: %s", err)} - } - for _, service := range dBResponse { - if service.ResponseStatusCode != "" { - traceFilterReponse.ResponseStatusCode[service.ResponseStatusCode] = service.Count - } - } - - default: - return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("filter type: %s not supported", e)} - } - } - - return &traceFilterReponse, nil -} - func getStatusFilters(query string, statusParams []string, excludeMap map[string]struct{}) string { // status can only be two and if both are selected than they are equivalent to none selected @@ -1120,140 +824,6 @@ func getStatusFilters(query string, statusParams []string, excludeMap map[string return query } -func (r *ClickHouseReader) GetFilteredSpans(ctx context.Context, queryParams *model.GetFilteredSpansParams) (*model.GetFilterSpansResponse, *model.ApiError) { - - queryTable := fmt.Sprintf("%s.%s", r.TraceDB, r.traceTableName) - - excludeMap := make(map[string]struct{}) - for _, e := range queryParams.Exclude { - if e == constants.OperationRequest { - excludeMap[constants.OperationDB] = struct{}{} - continue - } - excludeMap[e] = struct{}{} - } - - var query string - args := []interface{}{clickhouse.Named("timestampL", strconv.FormatInt(queryParams.Start.UnixNano(), 10)), clickhouse.Named("timestampU", strconv.FormatInt(queryParams.End.UnixNano(), 10))} - if len(queryParams.TraceID) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.TraceID, constants.TraceID, &query, args) - } - if len(queryParams.ServiceName) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.ServiceName, constants.ServiceName, &query, args) - } - if len(queryParams.HttpRoute) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpRoute, constants.HttpRoute, &query, args) - } - if len(queryParams.HttpHost) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpHost, constants.HttpHost, &query, args) - } - if len(queryParams.HttpMethod) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpMethod, constants.HttpMethod, &query, args) - } - if len(queryParams.HttpUrl) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpUrl, constants.HttpUrl, &query, args) - } - if len(queryParams.Operation) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.Operation, constants.OperationDB, &query, args) - } - if len(queryParams.RPCMethod) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.RPCMethod, constants.RPCMethod, &query, args) - } - - if len(queryParams.ResponseStatusCode) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.ResponseStatusCode, constants.ResponseStatusCode, &query, args) - } - - if len(queryParams.MinDuration) != 0 { - query = query + " AND durationNano >= @durationNanoMin" - args = append(args, clickhouse.Named("durationNanoMin", queryParams.MinDuration)) - } - if len(queryParams.MaxDuration) != 0 { - query = query + " AND durationNano <= @durationNanoMax" - args = append(args, clickhouse.Named("durationNanoMax", queryParams.MaxDuration)) - } - query = getStatusFilters(query, queryParams.Status, excludeMap) - - if len(queryParams.SpanKind) != 0 { - query = query + " AND kind = @kind" - args = append(args, clickhouse.Named("kind", queryParams.SpanKind)) - } - - // create TagQuery from TagQueryParams - tags := createTagQueryFromTagQueryParams(queryParams.Tags) - subQuery, argsSubQuery, errStatus := buildQueryWithTagParams(ctx, tags) - query += subQuery - args = append(args, argsSubQuery...) - if errStatus != nil { - return nil, errStatus - } - - if len(queryParams.OrderParam) != 0 { - if queryParams.OrderParam == constants.Duration { - queryTable = fmt.Sprintf("%s.%s", r.TraceDB, r.durationTable) - if queryParams.Order == constants.Descending { - query = query + " ORDER BY durationNano DESC" - } - if queryParams.Order == constants.Ascending { - query = query + " ORDER BY durationNano ASC" - } - } else if queryParams.OrderParam == constants.Timestamp { - projectionOptQuery := "SET allow_experimental_projection_optimization = 1" - err := r.db.Exec(ctx, projectionOptQuery) - - zap.L().Info(projectionOptQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query")} - } - if queryParams.Order == constants.Descending { - query = query + " ORDER BY timestamp DESC" - } - if queryParams.Order == constants.Ascending { - query = query + " ORDER BY timestamp ASC" - } - } - } - if queryParams.Limit > 0 { - query = query + " LIMIT @limit" - args = append(args, clickhouse.Named("limit", queryParams.Limit)) - } - - if queryParams.Offset > 0 { - query = query + " OFFSET @offset" - args = append(args, clickhouse.Named("offset", queryParams.Offset)) - } - - var getFilterSpansResponseItems []model.GetFilterSpansResponseItem - - baseQuery := fmt.Sprintf("SELECT timestamp, spanID, traceID, serviceName, name, durationNano, httpMethod, rpcMethod, responseStatusCode FROM %s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryTable) - baseQuery += query - err := r.db.Select(ctx, &getFilterSpansResponseItems, baseQuery, args...) - // Fill status and method - for i, e := range getFilterSpansResponseItems { - if e.RPCMethod != "" { - getFilterSpansResponseItems[i].Method = e.RPCMethod - } else { - getFilterSpansResponseItems[i].Method = e.HttpMethod - } - } - - zap.L().Info(baseQuery) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query")} - } - - getFilterSpansResponse := model.GetFilterSpansResponse{ - Spans: getFilterSpansResponseItems, - TotalSpans: 1000, - } - - return &getFilterSpansResponse, nil -} - func createTagQueryFromTagQueryParams(queryParams []model.TagQueryParam) []model.TagQuery { tags := []model.TagQuery{} for _, tag := range queryParams { @@ -1411,87 +981,6 @@ func addExistsOperator(item model.TagQuery, tagMapType string, not bool) (string return fmt.Sprintf(" AND %s (%s)", notStr, strings.Join(tagOperatorPair, " OR ")), args } -func (r *ClickHouseReader) GetTagFilters(ctx context.Context, queryParams *model.TagFilterParams) (*model.TagFilters, *model.ApiError) { - - excludeMap := make(map[string]struct{}) - for _, e := range queryParams.Exclude { - if e == constants.OperationRequest { - excludeMap[constants.OperationDB] = struct{}{} - continue - } - excludeMap[e] = struct{}{} - } - - var query string - args := []interface{}{clickhouse.Named("timestampL", strconv.FormatInt(queryParams.Start.UnixNano(), 10)), clickhouse.Named("timestampU", strconv.FormatInt(queryParams.End.UnixNano(), 10))} - if len(queryParams.TraceID) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.TraceID, constants.TraceID, &query, args) - } - if len(queryParams.ServiceName) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.ServiceName, constants.ServiceName, &query, args) - } - if len(queryParams.HttpRoute) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpRoute, constants.HttpRoute, &query, args) - } - if len(queryParams.HttpHost) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpHost, constants.HttpHost, &query, args) - } - if len(queryParams.HttpMethod) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpMethod, constants.HttpMethod, &query, args) - } - if len(queryParams.HttpUrl) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpUrl, constants.HttpUrl, &query, args) - } - if len(queryParams.Operation) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.Operation, constants.OperationDB, &query, args) - } - if len(queryParams.RPCMethod) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.RPCMethod, constants.RPCMethod, &query, args) - } - if len(queryParams.ResponseStatusCode) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.ResponseStatusCode, constants.ResponseStatusCode, &query, args) - } - if len(queryParams.MinDuration) != 0 { - query = query + " AND durationNano >= @durationNanoMin" - args = append(args, clickhouse.Named("durationNanoMin", queryParams.MinDuration)) - } - if len(queryParams.MaxDuration) != 0 { - query = query + " AND durationNano <= @durationNanoMax" - args = append(args, clickhouse.Named("durationNanoMax", queryParams.MaxDuration)) - } - if len(queryParams.SpanKind) != 0 { - query = query + " AND kind = @kind" - args = append(args, clickhouse.Named("kind", queryParams.SpanKind)) - } - - query = getStatusFilters(query, queryParams.Status, excludeMap) - - tagFilters := []model.TagFilters{} - - // Alternative finalQuery := fmt.Sprintf(`SELECT DISTINCT arrayJoin(tagMap.keys) as tagKeys FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, indexTable) - finalQuery := fmt.Sprintf(`SELECT groupUniqArrayArray(mapKeys(stringTagMap)) as stringTagKeys, groupUniqArrayArray(mapKeys(numberTagMap)) as numberTagKeys, groupUniqArrayArray(mapKeys(boolTagMap)) as boolTagKeys FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, r.traceTableName) - finalQuery += query - err := r.db.Select(ctx, &tagFilters, finalQuery, args...) - - zap.L().Info(query) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query")} - } - tagFiltersResult := model.TagFilters{ - StringTagKeys: make([]string, 0), - NumberTagKeys: make([]string, 0), - BoolTagKeys: make([]string, 0), - } - if len(tagFilters) != 0 { - tagFiltersResult.StringTagKeys = excludeTags(ctx, tagFilters[0].StringTagKeys) - tagFiltersResult.NumberTagKeys = excludeTags(ctx, tagFilters[0].NumberTagKeys) - tagFiltersResult.BoolTagKeys = excludeTags(ctx, tagFilters[0].BoolTagKeys) - } - return &tagFiltersResult, nil -} - func excludeTags(_ context.Context, tags []string) []string { excludedTagsMap := map[string]bool{ "http.code": true, @@ -1515,102 +1004,6 @@ func excludeTags(_ context.Context, tags []string) []string { return newTags } -func (r *ClickHouseReader) GetTagValues(ctx context.Context, queryParams *model.TagFilterParams) (*model.TagValues, *model.ApiError) { - - if queryParams.TagKey.Type == model.TagTypeNumber { - return &model.TagValues{ - NumberTagValues: make([]float64, 0), - StringTagValues: make([]string, 0), - BoolTagValues: make([]bool, 0), - }, nil - } else if queryParams.TagKey.Type == model.TagTypeBool { - return &model.TagValues{ - NumberTagValues: make([]float64, 0), - StringTagValues: make([]string, 0), - BoolTagValues: []bool{true, false}, - }, nil - } - - excludeMap := make(map[string]struct{}) - for _, e := range queryParams.Exclude { - if e == constants.OperationRequest { - excludeMap[constants.OperationDB] = struct{}{} - continue - } - excludeMap[e] = struct{}{} - } - - var query string - args := []interface{}{clickhouse.Named("timestampL", strconv.FormatInt(queryParams.Start.UnixNano(), 10)), clickhouse.Named("timestampU", strconv.FormatInt(queryParams.End.UnixNano(), 10))} - if len(queryParams.TraceID) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.TraceID, constants.TraceID, &query, args) - } - if len(queryParams.ServiceName) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.ServiceName, constants.ServiceName, &query, args) - } - if len(queryParams.HttpRoute) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpRoute, constants.HttpRoute, &query, args) - } - if len(queryParams.HttpHost) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpHost, constants.HttpHost, &query, args) - } - if len(queryParams.HttpMethod) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpMethod, constants.HttpMethod, &query, args) - } - if len(queryParams.HttpUrl) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpUrl, constants.HttpUrl, &query, args) - } - if len(queryParams.Operation) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.Operation, constants.OperationDB, &query, args) - } - if len(queryParams.MinDuration) != 0 { - query = query + " AND durationNano >= @durationNanoMin" - args = append(args, clickhouse.Named("durationNanoMin", queryParams.MinDuration)) - } - if len(queryParams.MaxDuration) != 0 { - query = query + " AND durationNano <= @durationNanoMax" - args = append(args, clickhouse.Named("durationNanoMax", queryParams.MaxDuration)) - } - if len(queryParams.SpanKind) != 0 { - query = query + " AND kind = @kind" - args = append(args, clickhouse.Named("kind", queryParams.SpanKind)) - } - - query = getStatusFilters(query, queryParams.Status, excludeMap) - - tagValues := []model.TagValues{} - - finalQuery := fmt.Sprintf(`SELECT groupArray(DISTINCT attributes_string[@key]) as stringTagValues FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU`, r.TraceDB, r.traceTableName) - finalQuery += query - finalQuery += " LIMIT @limit" - - args = append(args, clickhouse.Named("key", queryParams.TagKey.Key)) - args = append(args, clickhouse.Named("limit", queryParams.Limit)) - err := r.db.Select(ctx, &tagValues, finalQuery, args...) - - zap.L().Info(query) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query")} - } - - cleanedTagValues := model.TagValues{ - StringTagValues: []string{}, - NumberTagValues: []float64{}, - BoolTagValues: []bool{}, - } - if len(tagValues) == 0 { - return &cleanedTagValues, nil - } - for _, e := range tagValues[0].StringTagValues { - if e != "" { - cleanedTagValues.StringTagValues = append(cleanedTagValues.StringTagValues, e) - } - } - return &cleanedTagValues, nil -} - func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *model.GetTopOperationsParams) (*[]model.TopOperationsItem, *model.ApiError) { namedArgs := []interface{}{ @@ -1905,7 +1298,6 @@ func (r *ClickHouseReader) SearchTraces(ctx context.Context, params *model.Searc err = r.featureFlags.CheckFeature(model.SmartTraceDetail) smartAlgoEnabled := err == nil - // TODO(nitya): this will never run remove it if len(searchScanResponses) > params.SpansRenderLimit && smartAlgoEnabled { start = time.Now() searchSpansResult, err = smartTraceAlgorithm(searchSpanResponses, params.SpanID, params.LevelUp, params.LevelDown, params.SpansRenderLimit) @@ -1983,184 +1375,6 @@ func (r *ClickHouseReader) GetDependencyGraph(ctx context.Context, queryParams * return &response, nil } -func (r *ClickHouseReader) GetFilteredSpansAggregates(ctx context.Context, queryParams *model.GetFilteredSpanAggregatesParams) (*model.GetFilteredSpansAggregatesResponse, *model.ApiError) { - excludeMap := make(map[string]struct{}) - for _, e := range queryParams.Exclude { - if e == constants.OperationRequest { - excludeMap[constants.OperationDB] = struct{}{} - continue - } - excludeMap[e] = struct{}{} - } - - SpanAggregatesDBResponseItems := []model.SpanAggregatesDBResponseItem{} - - aggregation_query := "" - if queryParams.Dimension == "duration" { - switch queryParams.AggregationOption { - case "p50": - aggregation_query = " quantile(0.50)(durationNano) as float64Value " - case "p95": - aggregation_query = " quantile(0.95)(durationNano) as float64Value " - case "p90": - aggregation_query = " quantile(0.90)(durationNano) as float64Value " - case "p99": - aggregation_query = " quantile(0.99)(durationNano) as float64Value " - case "max": - aggregation_query = " max(durationNano) as value " - case "min": - aggregation_query = " min(durationNano) as value " - case "avg": - aggregation_query = " avg(durationNano) as float64Value " - case "sum": - aggregation_query = " sum(durationNano) as value " - default: - return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("aggregate type: %s not supported", queryParams.AggregationOption)} - } - } else if queryParams.Dimension == "calls" { - aggregation_query = " count(*) as value " - } - - args := []interface{}{clickhouse.Named("timestampL", strconv.FormatInt(queryParams.Start.UnixNano(), 10)), clickhouse.Named("timestampU", strconv.FormatInt(queryParams.End.UnixNano(), 10))} - - var query string - var customStr []string - _, columnExists := constants.GroupByColMap[queryParams.GroupBy] - // Using %s for groupBy params as it can be a custom column and custom columns are not supported by clickhouse-go yet: - // issue link: https://github.com/ClickHouse/clickhouse-go/issues/870 - if queryParams.GroupBy != "" && columnExists { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, queryParams.GroupBy, aggregation_query, r.TraceDB, r.traceTableName) - args = append(args, clickhouse.Named("groupByVar", queryParams.GroupBy)) - } else if queryParams.GroupBy != "" { - customStr = strings.Split(queryParams.GroupBy, ".(") - if len(customStr) < 2 { - return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("GroupBy: %s not supported", queryParams.GroupBy)} - } - if customStr[1] == string(model.TagTypeString)+")" { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, stringTagMap['%s'] as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, r.traceTableName) - } else if customStr[1] == string(model.TagTypeNumber)+")" { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(numberTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, r.traceTableName) - } else if customStr[1] == string(model.TagTypeBool)+")" { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, toString(boolTagMap['%s']) as groupBy, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, customStr[0], aggregation_query, r.TraceDB, r.traceTableName) - } else { - // return error for unsupported group by - return nil, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("GroupBy: %s not supported", queryParams.GroupBy)} - } - } else { - query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d minute) as time, %s FROM %s.%s WHERE timestamp >= @timestampL AND timestamp <= @timestampU", queryParams.StepSeconds/60, aggregation_query, r.TraceDB, r.traceTableName) - } - - if len(queryParams.TraceID) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.TraceID, constants.TraceID, &query, args) - } - if len(queryParams.ServiceName) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.ServiceName, constants.ServiceName, &query, args) - } - if len(queryParams.HttpRoute) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpRoute, constants.HttpRoute, &query, args) - } - if len(queryParams.HttpHost) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpHost, constants.HttpHost, &query, args) - } - if len(queryParams.HttpMethod) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpMethod, constants.HttpMethod, &query, args) - } - if len(queryParams.HttpUrl) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.HttpUrl, constants.HttpUrl, &query, args) - } - if len(queryParams.Operation) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.Operation, constants.OperationDB, &query, args) - } - if len(queryParams.RPCMethod) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.RPCMethod, constants.RPCMethod, &query, args) - } - if len(queryParams.ResponseStatusCode) > 0 { - args = buildFilterArrayQuery(ctx, excludeMap, queryParams.ResponseStatusCode, constants.ResponseStatusCode, &query, args) - } - if len(queryParams.MinDuration) != 0 { - query = query + " AND durationNano >= @durationNanoMin" - args = append(args, clickhouse.Named("durationNanoMin", queryParams.MinDuration)) - } - if len(queryParams.MaxDuration) != 0 { - query = query + " AND durationNano <= @durationNanoMax" - args = append(args, clickhouse.Named("durationNanoMax", queryParams.MaxDuration)) - } - query = getStatusFilters(query, queryParams.Status, excludeMap) - - if len(queryParams.SpanKind) != 0 { - query = query + " AND kind = @kind" - args = append(args, clickhouse.Named("kind", queryParams.SpanKind)) - } - // create TagQuery from TagQueryParams - tags := createTagQueryFromTagQueryParams(queryParams.Tags) - subQuery, argsSubQuery, errStatus := buildQueryWithTagParams(ctx, tags) - query += subQuery - args = append(args, argsSubQuery...) - - if errStatus != nil { - return nil, errStatus - } - - if queryParams.GroupBy != "" && columnExists { - query = query + fmt.Sprintf(" GROUP BY time, %s as groupBy ORDER BY time", queryParams.GroupBy) - } else if queryParams.GroupBy != "" { - if customStr[1] == string(model.TagTypeString)+")" { - query = query + fmt.Sprintf(" GROUP BY time, stringTagMap['%s'] as groupBy ORDER BY time", customStr[0]) - } else if customStr[1] == string(model.TagTypeNumber)+")" { - query = query + fmt.Sprintf(" GROUP BY time, toString(numberTagMap['%s']) as groupBy ORDER BY time", customStr[0]) - } else if customStr[1] == string(model.TagTypeBool)+")" { - query = query + fmt.Sprintf(" GROUP BY time, toString(boolTagMap['%s']) as groupBy ORDER BY time", customStr[0]) - } - } else { - query = query + " GROUP BY time ORDER BY time" - } - - err := r.db.Select(ctx, &SpanAggregatesDBResponseItems, query, args...) - - zap.L().Info(query) - - if err != nil { - zap.L().Error("Error in processing sql query", zap.Error(err)) - return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query")} - } - - GetFilteredSpansAggregatesResponse := model.GetFilteredSpansAggregatesResponse{ - Items: map[int64]model.SpanAggregatesResponseItem{}, - } - - for i := range SpanAggregatesDBResponseItems { - if SpanAggregatesDBResponseItems[i].Value == 0 { - SpanAggregatesDBResponseItems[i].Value = uint64(SpanAggregatesDBResponseItems[i].Float64Value) - } - SpanAggregatesDBResponseItems[i].Timestamp = int64(SpanAggregatesDBResponseItems[i].Time.UnixNano()) - SpanAggregatesDBResponseItems[i].FloatValue = float32(SpanAggregatesDBResponseItems[i].Value) - if queryParams.AggregationOption == "rate_per_sec" { - SpanAggregatesDBResponseItems[i].FloatValue = float32(SpanAggregatesDBResponseItems[i].Value) / float32(queryParams.StepSeconds) - } - if responseElement, ok := GetFilteredSpansAggregatesResponse.Items[SpanAggregatesDBResponseItems[i].Timestamp]; !ok { - if queryParams.GroupBy != "" && SpanAggregatesDBResponseItems[i].GroupBy != "" { - GetFilteredSpansAggregatesResponse.Items[SpanAggregatesDBResponseItems[i].Timestamp] = model.SpanAggregatesResponseItem{ - Timestamp: SpanAggregatesDBResponseItems[i].Timestamp, - GroupBy: map[string]float32{SpanAggregatesDBResponseItems[i].GroupBy: SpanAggregatesDBResponseItems[i].FloatValue}, - } - } else if queryParams.GroupBy == "" { - GetFilteredSpansAggregatesResponse.Items[SpanAggregatesDBResponseItems[i].Timestamp] = model.SpanAggregatesResponseItem{ - Timestamp: SpanAggregatesDBResponseItems[i].Timestamp, - Value: SpanAggregatesDBResponseItems[i].FloatValue, - } - } - - } else { - if queryParams.GroupBy != "" && SpanAggregatesDBResponseItems[i].GroupBy != "" { - responseElement.GroupBy[SpanAggregatesDBResponseItems[i].GroupBy] = SpanAggregatesDBResponseItems[i].FloatValue - } - GetFilteredSpansAggregatesResponse.Items[SpanAggregatesDBResponseItems[i].Timestamp] = responseElement - } - } - - return &GetFilteredSpansAggregatesResponse, nil -} - func getLocalTableName(tableName string) string { tableNameSplit := strings.Split(tableName, ".") diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 9c44e01508..43231f84d4 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -537,12 +537,6 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *AuthMiddleware) { router.HandleFunc("/api/v1/configs", am.OpenAccess(aH.getConfigs)).Methods(http.MethodGet) router.HandleFunc("/api/v1/health", am.OpenAccess(aH.getHealth)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/getSpanFilters", am.ViewAccess(aH.getSpanFilters)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getTagFilters", am.ViewAccess(aH.getTagFilters)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getFilteredSpans", am.ViewAccess(aH.getFilteredSpans)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getFilteredSpans/aggregates", am.ViewAccess(aH.getFilteredSpanAggregates)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getTagValues", am.ViewAccess(aH.getTagValues)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/listErrors", am.ViewAccess(aH.listErrors)).Methods(http.MethodPost) router.HandleFunc("/api/v1/countErrors", am.ViewAccess(aH.countErrors)).Methods(http.MethodPost) router.HandleFunc("/api/v1/errorFromErrorID", am.ViewAccess(aH.getErrorFromErrorID)).Methods(http.MethodGet) @@ -1858,86 +1852,6 @@ func (aH *APIHandler) getErrorFromGroupID(w http.ResponseWriter, r *http.Request aH.WriteJSON(w, r, result) } -func (aH *APIHandler) getSpanFilters(w http.ResponseWriter, r *http.Request) { - - query, err := parseSpanFilterRequestBody(r) - if aH.HandleError(w, err, http.StatusBadRequest) { - return - } - - result, apiErr := aH.reader.GetSpanFilters(r.Context(), query) - - if apiErr != nil && aH.HandleError(w, apiErr.Err, http.StatusInternalServerError) { - return - } - - aH.WriteJSON(w, r, result) -} - -func (aH *APIHandler) getFilteredSpans(w http.ResponseWriter, r *http.Request) { - - query, err := parseFilteredSpansRequest(r, aH) - if aH.HandleError(w, err, http.StatusBadRequest) { - return - } - - result, apiErr := aH.reader.GetFilteredSpans(r.Context(), query) - - if apiErr != nil && aH.HandleError(w, apiErr.Err, http.StatusInternalServerError) { - return - } - - aH.WriteJSON(w, r, result) -} - -func (aH *APIHandler) getFilteredSpanAggregates(w http.ResponseWriter, r *http.Request) { - - query, err := parseFilteredSpanAggregatesRequest(r) - if aH.HandleError(w, err, http.StatusBadRequest) { - return - } - - result, apiErr := aH.reader.GetFilteredSpansAggregates(r.Context(), query) - - if apiErr != nil && aH.HandleError(w, apiErr.Err, http.StatusInternalServerError) { - return - } - - aH.WriteJSON(w, r, result) -} - -func (aH *APIHandler) getTagFilters(w http.ResponseWriter, r *http.Request) { - - query, err := parseTagFilterRequest(r) - if aH.HandleError(w, err, http.StatusBadRequest) { - return - } - - result, apiErr := aH.reader.GetTagFilters(r.Context(), query) - - if apiErr != nil && aH.HandleError(w, apiErr.Err, http.StatusInternalServerError) { - return - } - - aH.WriteJSON(w, r, result) -} - -func (aH *APIHandler) getTagValues(w http.ResponseWriter, r *http.Request) { - - query, err := parseTagValueRequest(r) - if aH.HandleError(w, err, http.StatusBadRequest) { - return - } - - result, apiErr := aH.reader.GetTagValues(r.Context(), query) - - if apiErr != nil && aH.HandleError(w, apiErr.Err, http.StatusInternalServerError) { - return - } - - aH.WriteJSON(w, r, result) -} - func (aH *APIHandler) setTTL(w http.ResponseWriter, r *http.Request) { ttlParams, err := parseTTLParams(r) if aH.HandleError(w, err, http.StatusBadRequest) { diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index baad5f0a22..10c718aa28 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -29,15 +29,10 @@ type Reader interface { // GetDisks returns a list of disks configured in the underlying DB. It is supported by // clickhouse only. GetDisks(ctx context.Context) (*[]model.DiskItem, *model.ApiError) - GetSpanFilters(ctx context.Context, query *model.SpanFilterParams) (*model.SpanFiltersResponse, *model.ApiError) GetTraceAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error) GetTraceAttributeKeys(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) GetTraceAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) GetSpanAttributeKeys(ctx context.Context) (map[string]v3.AttributeKey, error) - GetTagFilters(ctx context.Context, query *model.TagFilterParams) (*model.TagFilters, *model.ApiError) - GetTagValues(ctx context.Context, query *model.TagFilterParams) (*model.TagValues, *model.ApiError) - GetFilteredSpans(ctx context.Context, query *model.GetFilteredSpansParams) (*model.GetFilterSpansResponse, *model.ApiError) - GetFilteredSpansAggregates(ctx context.Context, query *model.GetFilteredSpanAggregatesParams) (*model.GetFilteredSpansAggregatesResponse, *model.ApiError) ListErrors(ctx context.Context, params *model.ListErrorsParams) (*[]model.Error, *model.ApiError) CountErrors(ctx context.Context, params *model.CountErrorsParams) (uint64, *model.ApiError) From 71dccae8562a37df6079e90b2cbbddf32da948aa Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 18 Nov 2024 00:33:43 +0530 Subject: [PATCH 29/41] fix: attribute and attribute values v2 --- .../app/clickhouseReader/reader.go | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index fbd53da329..ef28910d80 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3958,7 +3958,63 @@ func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req return &response, nil } +func (r *ClickHouseReader) GetTraceAttributeKeysV2(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { + + var query string + var err error + var rows driver.Rows + var response v3.FilterAttributeKeyResponse + + query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, dataType FROM %s.%s WHERE tagKey ILIKE $1 LIMIT $2", r.TraceDB, r.spanAttributeTable) + + rows, err = r.db.Query(ctx, query, fmt.Sprintf("%%%s%%", req.SearchText), req.Limit) + + if err != nil { + zap.L().Error("Error while executing query", zap.Error(err)) + return nil, fmt.Errorf("error while executing query: %s", err.Error()) + } + defer rows.Close() + + statements := []model.ShowCreateTableStatement{} + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.TraceDB, r.traceLocalTableName) + err = r.db.Select(ctx, &statements, query) + if err != nil { + return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) + } + + var tagKey string + var dataType string + var tagType string + for rows.Next() { + if err := rows.Scan(&tagKey, &tagType, &dataType); err != nil { + return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) + } + key := v3.AttributeKey{ + Key: tagKey, + DataType: v3.AttributeKeyDataType(dataType), + Type: v3.AttributeKeyType(tagType), + IsColumn: isColumn(true, statements[0].Statement, tagType, tagKey, dataType), + } + response.AttributeKeys = append(response.AttributeKeys, key) + } + + // add other attributes + for _, f := range constants.StaticFieldsTraces { + if (v3.AttributeKey{} == f) { + continue + } + if len(req.SearchText) == 0 || strings.Contains(f.Key, req.SearchText) { + response.AttributeKeys = append(response.AttributeKeys, f) + } + } + + return &response, nil +} + func (r *ClickHouseReader) GetTraceAttributeKeys(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { + if r.useTraceNewSchema { + return r.GetTraceAttributeKeysV2(ctx, req) + } var query string var err error @@ -3997,8 +4053,88 @@ func (r *ClickHouseReader) GetTraceAttributeKeys(ctx context.Context, req *v3.Fi return &response, nil } +func (r *ClickHouseReader) GetTraceAttributeValuesV2(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { + var query string + var filterValueColumn string + var err error + var rows driver.Rows + var attributeValues v3.FilterAttributeValueResponse + // if dataType or tagType is not present return empty response + if len(req.FilterAttributeKeyDataType) == 0 || len(req.TagType) == 0 { + return &v3.FilterAttributeValueResponse{}, nil + } + + // if data type is bool, return true and false + if req.FilterAttributeKeyDataType == v3.AttributeKeyDataTypeBool { + return &v3.FilterAttributeValueResponse{ + BoolAttributeValues: []bool{true, false}, + }, nil + } + + query = "select distinct" + switch req.FilterAttributeKeyDataType { + case v3.AttributeKeyDataTypeFloat64: + filterValueColumn = "float64TagValue" + case v3.AttributeKeyDataTypeString: + filterValueColumn = "stringTagValue" + } + + searchText := fmt.Sprintf("%%%s%%", req.SearchText) + + // check if the tagKey is a topLevelColumn + if _, ok := constants.StaticFieldsTraces[req.FilterAttributeKey]; ok { + // query the column for the last 48 hours + filterValueColumnWhere := req.FilterAttributeKey + selectKey := req.FilterAttributeKey + if req.FilterAttributeKeyDataType != v3.AttributeKeyDataTypeString { + filterValueColumnWhere = fmt.Sprintf("toString(%s)", req.FilterAttributeKey) + selectKey = fmt.Sprintf("toInt64(%s)", req.FilterAttributeKey) + } + + query = fmt.Sprintf("select distinct %s from %s.%s where ts_bucket_start >= toUInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)) AND timestamp >= toDateTime64(now() - INTERVAL 48 HOUR, 9) and %s ILIKE $1 limit $2", selectKey, r.TraceDB, r.traceLocalTableName, filterValueColumnWhere) + rows, err = r.db.Query(ctx, query, searchText, req.Limit) + } else { + filterValueColumnWhere := filterValueColumn + if req.FilterAttributeKeyDataType != v3.AttributeKeyDataTypeString { + filterValueColumnWhere = fmt.Sprintf("toString(%s)", filterValueColumn) + } + query = fmt.Sprintf("select distinct %s from %s.%s where tagKey=$1 and %s ILIKE $2 and tagType=$3 limit $4", filterValueColumn, r.TraceDB, r.spanAttributeTable, filterValueColumnWhere) + rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, searchText, req.TagType, req.Limit) + } + + if err != nil { + zap.L().Error("Error while executing query", zap.Error(err)) + return nil, fmt.Errorf("error while executing query: %s", err.Error()) + } + defer rows.Close() + + var strAttributeValue string + var float64AttributeValue sql.NullFloat64 + for rows.Next() { + switch req.FilterAttributeKeyDataType { + case v3.AttributeKeyDataTypeFloat64: + if err := rows.Scan(&float64AttributeValue); err != nil { + return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) + } + if float64AttributeValue.Valid { + attributeValues.NumberAttributeValues = append(attributeValues.NumberAttributeValues, float64AttributeValue.Float64) + } + case v3.AttributeKeyDataTypeString: + if err := rows.Scan(&strAttributeValue); err != nil { + return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) + } + attributeValues.StringAttributeValues = append(attributeValues.StringAttributeValues, strAttributeValue) + } + } + + return &attributeValues, nil +} + func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { + if r.useTraceNewSchema { + return r.GetTraceAttributeValuesV2(ctx, req) + } var query string var err error var rows driver.Rows From d16fa05d11bfbda52b9fe1c6f2caeead31cb3f57 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 18 Nov 2024 15:04:21 +0530 Subject: [PATCH 30/41] fix: autocomplete api's updated --- .../app/clickhouseReader/reader.go | 103 +++++++- pkg/query-service/constants/constants.go | 232 ++++++++++++------ 2 files changed, 253 insertions(+), 82 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index ef28910d80..74e84e933f 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3890,7 +3890,99 @@ func (r *ClickHouseReader) CheckClickHouse(ctx context.Context) error { return nil } +func (r *ClickHouseReader) GetTraceAggregateAttributesV2(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error) { + var query string + var err error + var rows driver.Rows + var response v3.AggregateAttributeResponse + var stringAllowed bool + + where := "" + switch req.Operator { + case + v3.AggregateOperatorCountDistinct, + v3.AggregateOperatorCount: + where = "tagKey ILIKE $1" + stringAllowed = true + case + v3.AggregateOperatorRateSum, + v3.AggregateOperatorRateMax, + v3.AggregateOperatorRateAvg, + v3.AggregateOperatorRate, + v3.AggregateOperatorRateMin, + v3.AggregateOperatorP05, + v3.AggregateOperatorP10, + v3.AggregateOperatorP20, + v3.AggregateOperatorP25, + v3.AggregateOperatorP50, + v3.AggregateOperatorP75, + v3.AggregateOperatorP90, + v3.AggregateOperatorP95, + v3.AggregateOperatorP99, + v3.AggregateOperatorAvg, + v3.AggregateOperatorSum, + v3.AggregateOperatorMin, + v3.AggregateOperatorMax: + where = "tagKey ILIKE $1 AND dataType='float64'" + stringAllowed = false + case + v3.AggregateOperatorNoOp: + return &v3.AggregateAttributeResponse{}, nil + default: + return nil, fmt.Errorf("unsupported aggregate operator") + } + query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, dataType FROM %s.%s WHERE %s", r.TraceDB, r.spanAttributeTable, where) + if req.Limit != 0 { + query = query + fmt.Sprintf(" LIMIT %d;", req.Limit) + } + rows, err = r.db.Query(ctx, query, fmt.Sprintf("%%%s%%", req.SearchText)) + + if err != nil { + zap.L().Error("Error while executing query", zap.Error(err)) + return nil, fmt.Errorf("error while executing query: %s", err.Error()) + } + defer rows.Close() + + statements := []model.ShowCreateTableStatement{} + query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.TraceDB, r.traceLocalTableName) + err = r.db.Select(ctx, &statements, query) + if err != nil { + return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) + } + + var tagKey string + var dataType string + var tagType string + for rows.Next() { + if err := rows.Scan(&tagKey, &tagType, &dataType); err != nil { + return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) + } + key := v3.AttributeKey{ + Key: tagKey, + DataType: v3.AttributeKeyDataType(dataType), + Type: v3.AttributeKeyType(tagType), + IsColumn: isColumn(true, statements[0].Statement, tagType, tagKey, dataType), + } + response.AttributeKeys = append(response.AttributeKeys, key) + } + + // add other attributes + for _, field := range constants.NewStaticFieldsTraces { + if (!stringAllowed && field.DataType == v3.AttributeKeyDataTypeString) || (v3.AttributeKey{} == field) { + continue + } else if len(req.SearchText) == 0 || strings.Contains(field.Key, req.SearchText) { + response.AttributeKeys = append(response.AttributeKeys, field) + } + } + + return &response, nil +} + func (r *ClickHouseReader) GetTraceAggregateAttributes(ctx context.Context, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error) { + if r.useTraceNewSchema { + return r.GetTraceAggregateAttributesV2(ctx, req) + } + var query string var err error var rows driver.Rows @@ -3999,7 +4091,7 @@ func (r *ClickHouseReader) GetTraceAttributeKeysV2(ctx context.Context, req *v3. } // add other attributes - for _, f := range constants.StaticFieldsTraces { + for _, f := range constants.NewStaticFieldsTraces { if (v3.AttributeKey{} == f) { continue } @@ -4059,9 +4151,15 @@ func (r *ClickHouseReader) GetTraceAttributeValuesV2(ctx context.Context, req *v var err error var rows driver.Rows var attributeValues v3.FilterAttributeValueResponse + // if dataType or tagType is not present return empty response if len(req.FilterAttributeKeyDataType) == 0 || len(req.TagType) == 0 { - return &v3.FilterAttributeValueResponse{}, nil + // add data type if it's a top level key + if k, ok := constants.StaticFieldsTraces[req.FilterAttributeKey]; ok { + req.FilterAttributeKeyDataType = k.DataType + } else { + return &v3.FilterAttributeValueResponse{}, nil + } } // if data type is bool, return true and false @@ -4082,6 +4180,7 @@ func (r *ClickHouseReader) GetTraceAttributeValuesV2(ctx context.Context, req *v searchText := fmt.Sprintf("%%%s%%", req.SearchText) // check if the tagKey is a topLevelColumn + // here we are using StaticFieldsTraces instead of NewStaticFieldsTraces as we want to consider old columns as well. if _, ok := constants.StaticFieldsTraces[req.FilterAttributeKey]; ok { // query the column for the last 48 hours filterValueColumnWhere := req.FilterAttributeKey diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 1c03234a33..7072f13173 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -1,6 +1,7 @@ package constants import ( + "maps" "os" "strconv" "testing" @@ -447,30 +448,35 @@ const MaxFilterSuggestionsExamplesLimit = 10 var SpanRenderLimitStr = GetOrDefaultEnv("SPAN_RENDER_LIMIT", "2500") var MaxSpansInTraceStr = GetOrDefaultEnv("MAX_SPANS_IN_TRACE", "250000") -var StaticFieldsTraces = map[string]v3.AttributeKey{ +var NewStaticFieldsTraces = map[string]v3.AttributeKey{ "timestamp": {}, - "traceID": { - Key: "traceID", + "trace_id": { + Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "spanID": { - Key: "spanID", + "span_id": { + Key: "span_id", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "parentSpanID": { - Key: "parentSpanID", + "trace_state": { + Key: "trace_state", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "name": { - Key: "name", + "parent_span_id": { + Key: "parent_span_id", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "serviceName": { - Key: "serviceName", + "flags": { + Key: "flags", + DataType: v3.AttributeKeyDataTypeInt64, + IsColumn: true, + }, + "name": { + Key: "name", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, @@ -479,170 +485,236 @@ var StaticFieldsTraces = map[string]v3.AttributeKey{ DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "spanKind": { - Key: "spanKind", + "kind_string": { + Key: "kind_string", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "durationNano": { - Key: "durationNano", + "duration_nano": { + Key: "duration_nano", DataType: v3.AttributeKeyDataTypeFloat64, IsColumn: true, }, - "statusCode": { - Key: "statusCode", + "status_code": { + Key: "status_code", DataType: v3.AttributeKeyDataTypeFloat64, IsColumn: true, }, - "hasError": { - Key: "hasError", - DataType: v3.AttributeKeyDataTypeBool, + "status_message": { + Key: "status_message", + DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "statusMessage": { - Key: "statusMessage", + "status_code_string": { + Key: "status_code_string", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "statusCodeString": { - Key: "statusCodeString", + + // new support for composite attributes + "response_status_code": { + Key: "response_status_code", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "externalHttpMethod": { - Key: "externalHttpMethod", + "external_http_url": { + Key: "external_http_url", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "externalHttpUrl": { - Key: "externalHttpUrl", + "http_url": { + Key: "http_url", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "dbSystem": { - Key: "dbSystem", + "external_http_method": { + Key: "external_http_method", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "dbName": { - Key: "dbName", + "http_method": { + Key: "http_method", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "dbOperation": { - Key: "dbOperation", + "http_host": { + Key: "http_host", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "peerService": { - Key: "peerService", + "db_name": { + Key: "db_name", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "httpMethod": { - Key: "httpMethod", + "db_operation": { + Key: "db_operation", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "httpUrl": { - Key: "httpUrl", - DataType: v3.AttributeKeyDataTypeString, + "has_error": { + Key: "has_error", + DataType: v3.AttributeKeyDataTypeBool, IsColumn: true, }, - "httpRoute": { - Key: "httpRoute", + "is_remote": { + Key: "is_remote", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "httpHost": { - Key: "httpHost", + // the simple attributes are not present here as + // they are taken care by new format __'' +} + +var DeprecatedStaticFieldsTraces = map[string]v3.AttributeKey{ + "traceID": { + Key: "traceID", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "msgSystem": { - Key: "msgSystem", + "spanID": { + Key: "spanID", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "msgOperation": { - Key: "msgOperation", + "parentSpanID": { + Key: "parentSpanID", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "rpcSystem": { - Key: "rpcSystem", + "spanKind": { + Key: "spanKind", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "rpcService": { - Key: "rpcService", + "durationNano": { + Key: "durationNano", + DataType: v3.AttributeKeyDataTypeFloat64, + IsColumn: true, + }, + "statusCode": { + Key: "statusCode", + DataType: v3.AttributeKeyDataTypeFloat64, + IsColumn: true, + }, + "statusMessage": { + Key: "statusMessage", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "rpcMethod": { - Key: "rpcMethod", + "statusCodeString": { + Key: "statusCodeString", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, + + // old support for composite attributes "responseStatusCode": { Key: "responseStatusCode", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - - // new support - "response_status_code": { - Key: "response_status_code", + "externalHttpUrl": { + Key: "externalHttpUrl", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "external_http_url": { - Key: "external_http_url", + "httpUrl": { + Key: "httpUrl", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "http_url": { - Key: "http_url", + "externalHttpMethod": { + Key: "externalHttpMethod", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "external_http_method": { - Key: "external_http_method", + "httpMethod": { + Key: "httpMethod", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "http_method": { - Key: "http_method", + "httpHost": { + Key: "httpHost", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "http_host": { - Key: "http_host", + "dbName": { + Key: "dbName", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "db_name": { - Key: "db_name", + "dbOperation": { + Key: "dbOperation", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "db_operation": { - Key: "db_operation", + "hasError": { + Key: "hasError", + DataType: v3.AttributeKeyDataTypeBool, + IsColumn: true, + }, + "isRemote": { + Key: "isRemote", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "has_error": { - Key: "has_error", - DataType: v3.AttributeKeyDataTypeBool, + + // old support for resource attributes + "serviceName": { + Key: "serviceName", + DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - "is_remote": { - Key: "is_remote", + + // old support for simple attributes + "httpRoute": { + Key: "httpRoute", DataType: v3.AttributeKeyDataTypeString, IsColumn: true, }, - // the simple attributes are not present here as - // they are taken care by new format __'' + "msgSystem": { + Key: "msgSystem", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "msgOperation": { + Key: "msgOperation", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "dbSystem": { + Key: "dbSystem", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "rpcSystem": { + Key: "rpcSystem", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "rpcService": { + Key: "rpcService", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "rpcMethod": { + Key: "rpcMethod", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, + "peerService": { + Key: "peerService", + DataType: v3.AttributeKeyDataTypeString, + IsColumn: true, + }, +} + +var StaticFieldsTraces = map[string]v3.AttributeKey{} + +func init() { + StaticFieldsTraces = maps.Clone(NewStaticFieldsTraces) + maps.Copy(StaticFieldsTraces, DeprecatedStaticFieldsTraces) } const TRACE_V4_MAX_PAGINATION_LIMIT = 10000 From d4a17621519cb194e7f5158ca343f2c114822055 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 18 Nov 2024 15:37:00 +0530 Subject: [PATCH 31/41] fix: tests fixed --- pkg/query-service/app/clickhouseReader/reader.go | 6 +++--- pkg/query-service/rules/threshold_rule_test.go | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 74e84e933f..77ee6ba2d6 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3947,7 +3947,7 @@ func (r *ClickHouseReader) GetTraceAggregateAttributesV2(ctx context.Context, re query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.TraceDB, r.traceLocalTableName) err = r.db.Select(ctx, &statements, query) if err != nil { - return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) + return nil, fmt.Errorf("error while fetching trace schema: %s", err.Error()) } var tagKey string @@ -4071,7 +4071,7 @@ func (r *ClickHouseReader) GetTraceAttributeKeysV2(ctx context.Context, req *v3. query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.TraceDB, r.traceLocalTableName) err = r.db.Select(ctx, &statements, query) if err != nil { - return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) + return nil, fmt.Errorf("error while fetching trace schema: %s", err.Error()) } var tagKey string @@ -4305,7 +4305,7 @@ func (r *ClickHouseReader) GetSpanAttributeKeys(ctx context.Context) (map[string query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.TraceDB, r.traceTableName) err = r.db.Select(ctx, &statements, query) if err != nil { - return nil, fmt.Errorf("error while fetching logs schema: %s", err.Error()) + return nil, fmt.Errorf("error while fetching trace schema: %s", err.Error()) } var tagKey string diff --git a/pkg/query-service/rules/threshold_rule_test.go b/pkg/query-service/rules/threshold_rule_test.go index 30daf9de56..2582ae56fc 100644 --- a/pkg/query-service/rules/threshold_rule_test.go +++ b/pkg/query-service/rules/threshold_rule_test.go @@ -2,6 +2,7 @@ package rules import ( "context" + "fmt" "strings" "testing" "time" @@ -1118,7 +1119,9 @@ func TestThresholdRuleClickHouseTmpl(t *testing.T) { type queryMatcherAny struct { } -func (m *queryMatcherAny) Match(string, string) error { +func (m *queryMatcherAny) Match(x string, y string) error { + fmt.Println(x) + fmt.Println(y) return nil } @@ -1423,9 +1426,12 @@ func TestThresholdRuleTracesLink(t *testing.T) { for idx, c := range testCases { metaRows := cmock.NewRows(metaCols, c.metaValues) mock. - ExpectQuery("SELECT DISTINCT(tagKey), tagType, dataType, isColumn FROM archiveNamespace.span_attributes_keys"). + ExpectQuery("SELECT DISTINCT(tagKey), tagType, dataType FROM archiveNamespace.span_attributes_keys"). WillReturnRows(metaRows) + mock. + ExpectSelect("SHOW CREATE TABLE signoz_traces.distributed_signoz_index_v3").WillReturnRows(&cmock.Rows{}) + rows := cmock.NewRows(cols, c.values) // We are testing the eval logic after the query is run From b9cff1d8aff174aa41b6f4fca505acbd316e4db1 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 18 Nov 2024 17:13:02 +0530 Subject: [PATCH 32/41] feat: minor fixes --- .../app/clickhouseReader/reader.go | 57 ++++++++++++++++++- pkg/query-service/app/http_handler.go | 6 +- pkg/query-service/rules/threshold_rule.go | 9 ++- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 77ee6ba2d6..9a456aa772 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -3963,7 +3963,10 @@ func (r *ClickHouseReader) GetTraceAggregateAttributesV2(ctx context.Context, re Type: v3.AttributeKeyType(tagType), IsColumn: isColumn(true, statements[0].Statement, tagType, tagKey, dataType), } - response.AttributeKeys = append(response.AttributeKeys, key) + + if _, ok := constants.DeprecatedStaticFieldsTraces[tagKey]; !ok { + response.AttributeKeys = append(response.AttributeKeys, key) + } } // add other attributes @@ -4087,7 +4090,11 @@ func (r *ClickHouseReader) GetTraceAttributeKeysV2(ctx context.Context, req *v3. Type: v3.AttributeKeyType(tagType), IsColumn: isColumn(true, statements[0].Statement, tagType, tagKey, dataType), } - response.AttributeKeys = append(response.AttributeKeys, key) + + // don't send deprecated static fields + if _, ok := constants.DeprecatedStaticFieldsTraces[tagKey]; !ok { + response.AttributeKeys = append(response.AttributeKeys, key) + } } // add other attributes @@ -4286,7 +4293,7 @@ func (r *ClickHouseReader) GetTraceAttributeValues(ctx context.Context, req *v3. return &attributeValues, nil } -func (r *ClickHouseReader) GetSpanAttributeKeys(ctx context.Context) (map[string]v3.AttributeKey, error) { +func (r *ClickHouseReader) GetSpanAttributeKeysV2(ctx context.Context) (map[string]v3.AttributeKey, error) { var query string var err error var rows driver.Rows @@ -4325,6 +4332,50 @@ func (r *ClickHouseReader) GetSpanAttributeKeys(ctx context.Context) (map[string name := tagKey + "##" + tagType + "##" + strings.ToLower(dataType) response[name] = key } + + for _, key := range constants.StaticFieldsTraces { + name := key.Key + "##" + key.Type.String() + "##" + strings.ToLower(key.DataType.String()) + response[name] = key + } + + return response, nil +} + +func (r *ClickHouseReader) GetSpanAttributeKeys(ctx context.Context) (map[string]v3.AttributeKey, error) { + if r.useTraceNewSchema { + return r.GetSpanAttributeKeysV2(ctx) + } + var query string + var err error + var rows driver.Rows + response := map[string]v3.AttributeKey{} + + query = fmt.Sprintf("SELECT DISTINCT(tagKey), tagType, dataType, isColumn FROM %s.%s", r.TraceDB, r.spanAttributesKeysTable) + + rows, err = r.db.Query(ctx, query) + + if err != nil { + zap.L().Error("Error while executing query", zap.Error(err)) + return nil, fmt.Errorf("error while executing query: %s", err.Error()) + } + defer rows.Close() + + var tagKey string + var dataType string + var tagType string + var isColumn bool + for rows.Next() { + if err := rows.Scan(&tagKey, &tagType, &dataType, &isColumn); err != nil { + return nil, fmt.Errorf("error while scanning rows: %s", err.Error()) + } + key := v3.AttributeKey{ + Key: tagKey, + DataType: v3.AttributeKeyDataType(dataType), + Type: v3.AttributeKeyType(tagType), + IsColumn: isColumn, + } + response[tagKey] = key + } return response, nil } diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 43231f84d4..7286ee501a 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -4833,7 +4833,11 @@ func (aH *APIHandler) queryRangeV4(ctx context.Context, queryRangeParams *v3.Que RespondError(w, apiErrObj, errQuriesByName) return } - tracesV3.Enrich(queryRangeParams, spanKeys) + if aH.UseTraceNewSchema { + tracesV4.Enrich(queryRangeParams, spanKeys) + } else { + tracesV3.Enrich(queryRangeParams, spanKeys) + } } // WARN: Only works for AND operator in traces query diff --git a/pkg/query-service/rules/threshold_rule.go b/pkg/query-service/rules/threshold_rule.go index 1c0d475983..ad55083f8f 100644 --- a/pkg/query-service/rules/threshold_rule.go +++ b/pkg/query-service/rules/threshold_rule.go @@ -29,6 +29,7 @@ import ( logsv3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" + tracesV4 "go.signoz.io/signoz/pkg/query-service/app/traces/v4" "go.signoz.io/signoz/pkg/query-service/formatter" yaml "gopkg.in/yaml.v2" @@ -50,6 +51,8 @@ type ThresholdRule struct { // used for attribute metadata enrichment for logs and traces logsKeys map[string]v3.AttributeKey spansKeys map[string]v3.AttributeKey + + useTraceNewSchema bool } func NewThresholdRule( @@ -299,7 +302,11 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, ts time.Time) (Vec return nil, err } r.spansKeys = spanKeys - tracesV3.Enrich(params, spanKeys) + if r.useTraceNewSchema { + tracesV4.Enrich(params, spanKeys) + } else { + tracesV3.Enrich(params, spanKeys) + } } } From fa77efb02f04a972af2efaa9a9fdb7344a528e07 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 18 Nov 2024 17:42:32 +0530 Subject: [PATCH 33/41] fix: update telemetry functions --- .../app/clickhouseReader/reader.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 9a456aa772..f5c37f19cf 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -487,10 +487,10 @@ func (r *ClickHouseReader) GetQueryRangeResult(ctx context.Context, query *model func (r *ClickHouseReader) GetServicesList(ctx context.Context) (*[]string, error) { services := []string{} - query := fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceLocalTableName) + query := fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceTableName) if r.useTraceNewSchema { - query = fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE ts_bucket_start > (toUnixTimestamp(now()) - 1800) AND toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceLocalTableName) + query = fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE ts_bucket_start > (toUnixTimestamp(now()) - 1800) AND toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceTableName) } rows, err := r.db.Query(ctx, query) @@ -2299,7 +2299,7 @@ func (r *ClickHouseReader) GetTotalSpans(ctx context.Context) (uint64, error) { var totalSpans uint64 - queryStr := fmt.Sprintf("SELECT count() from %s.%s;", signozTraceDBName, signozTraceTableName) + queryStr := fmt.Sprintf("SELECT count() from %s.%s;", signozTraceDBName, r.traceTableName) r.db.QueryRow(ctx, queryStr).Scan(&totalSpans) return totalSpans, nil @@ -2310,7 +2310,9 @@ func (r *ClickHouseReader) GetSpansInLastHeartBeatInterval(ctx context.Context, var spansInLastHeartBeatInterval uint64 queryStr := fmt.Sprintf("SELECT count() from %s.%s where timestamp > toUnixTimestamp(now()-toIntervalMinute(%d));", signozTraceDBName, signozSpansTable, int(interval.Minutes())) - + if r.useTraceNewSchema { + queryStr = fmt.Sprintf("SELECT count() from %s.%s where ts_bucket_start >= toUInt64(toUnixTimestamp(now() - toIntervalMinute(%d))) - 1800 and timestamp > toUnixTimestamp(now()-toIntervalMinute(%d));", signozTraceDBName, r.traceTableName, int(interval.Minutes()), int(interval.Minutes())) + } r.db.QueryRow(ctx, queryStr).Scan(&spansInLastHeartBeatInterval) return spansInLastHeartBeatInterval, nil @@ -2434,6 +2436,13 @@ func (r *ClickHouseReader) GetTagsInfoInLastHeartBeatInterval(ctx context.Contex where timestamp > toUnixTimestamp(now()-toIntervalMinute(%d)) group by serviceName, env, language;`, r.TraceDB, r.traceTableName, int(interval.Minutes())) + if r.useTraceNewSchema { + queryStr = fmt.Sprintf(`select serviceName, resources_string['deployment.environment'] as env, + resources_string['telemetry.sdk.language'] as language from %s.%s + where timestamp > toUnixTimestamp(now()-toIntervalMinute(%d)) + group by serviceName, env, language;`, r.TraceDB, r.traceTableName, int(interval.Minutes())) + } + tagTelemetryDataList := []model.TagTelemetryData{} err := r.db.Select(ctx, &tagTelemetryDataList, queryStr) @@ -4197,7 +4206,7 @@ func (r *ClickHouseReader) GetTraceAttributeValuesV2(ctx context.Context, req *v selectKey = fmt.Sprintf("toInt64(%s)", req.FilterAttributeKey) } - query = fmt.Sprintf("select distinct %s from %s.%s where ts_bucket_start >= toUInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)) AND timestamp >= toDateTime64(now() - INTERVAL 48 HOUR, 9) and %s ILIKE $1 limit $2", selectKey, r.TraceDB, r.traceLocalTableName, filterValueColumnWhere) + query = fmt.Sprintf("select distinct %s from %s.%s where ts_bucket_start >= toUInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)) AND timestamp >= toDateTime64(now() - INTERVAL 48 HOUR, 9) and %s ILIKE $1 limit $2", selectKey, r.TraceDB, r.traceTableName, filterValueColumnWhere) rows, err = r.db.Query(ctx, query, searchText, req.Limit) } else { filterValueColumnWhere := filterValueColumn From 15ace5da17cfcab6dd3182e938f2aaa9cb9eee0e Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Mon, 18 Nov 2024 22:26:23 +0530 Subject: [PATCH 34/41] fix: dont use alias, use proper col names --- pkg/query-service/app/clickhouseReader/reader.go | 4 ++-- pkg/query-service/model/response.go | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index f5c37f19cf..0777cb3ebf 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -1101,7 +1101,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea } var traceSummary model.TraceSummary - summaryQuery := fmt.Sprintf("SELECT * from %s.%s WHERE traceID=$1", r.TraceDB, r.traceSummaryTable) + summaryQuery := fmt.Sprintf("SELECT * from %s.%s WHERE trace_id=$1", r.TraceDB, r.traceSummaryTable) err := r.db.QueryRow(ctx, summaryQuery, params.TraceID).Scan(&traceSummary.TraceID, &traceSummary.Start, &traceSummary.End, &traceSummary.NumSpans) if err != nil { if err == sql.ErrNoRows { @@ -1136,7 +1136,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea var startTime, endTime, durationNano uint64 var searchScanResponses []model.SearchSpanResponseItemV2 - query := fmt.Sprintf("SELECT timestamp, durationNano, spanID, traceID, hasError, kind, serviceName, name, references, attributes_string, events, statusMessage, statusCodeString, spanKind FROM %s.%s WHERE traceID=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceTableName) + query := fmt.Sprintf("SELECT timestamp, duration_nano, span_id, trace_id, has_error, kind, resource_string_service$$name, name, references, attributes_string, events, status_message, status_code_string, kind_string FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceTableName) start := time.Now() diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 8421b11328..2d86dfa862 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -271,21 +271,21 @@ type SearchSpanResponseItem struct { type SearchSpanResponseItemV2 struct { TimeUnixNano time.Time `json:"timestamp" ch:"timestamp"` - DurationNano uint64 `json:"durationNano" ch:"durationNano"` - SpanID string `json:"spanId" ch:"spanID"` - TraceID string `json:"traceId" ch:"traceID"` - HasError bool `json:"hasError" ch:"hasError"` + DurationNano uint64 `json:"durationNano" ch:"duration_nano"` + SpanID string `json:"spanId" ch:"span_id"` + TraceID string `json:"traceId" ch:"trace_id"` + HasError bool `json:"hasError" ch:"has_error"` Kind int8 `json:"kind" ch:"kind"` - ServiceName string `json:"serviceName" ch:"serviceName"` + ServiceName string `json:"serviceName" ch:"resource_string_service$$name"` Name string `json:"name" ch:"name"` References string `json:"references,omitempty" ch:"references"` Attributes_string map[string]string `json:"attributes_string" ch:"attributes_string"` Attributes_number map[string]float64 `json:"attributes_number" ch:"attributes_number"` Attributes_bool map[string]bool `json:"attributes_bool" ch:"attributes_bool"` Events []string `json:"event" ch:"events"` - StatusMessage string `json:"statusMessage" ch:"statusMessage"` - StatusCodeString string `json:"statusCodeString" ch:"statusCodeString"` - SpanKind string `json:"spanKind" ch:"spanKind"` + StatusMessage string `json:"statusMessage" ch:"status_message"` + StatusCodeString string `json:"statusCodeString" ch:"status_code_string"` + SpanKind string `json:"spanKind" ch:"kind_string"` } type TraceSummary struct { From d87c1f00758ca4f7f4633171dd9b3b0a1775feae Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 19 Nov 2024 14:18:31 +0530 Subject: [PATCH 35/41] fix: move models to it's own file --- .../app/clickhouseReader/reader.go | 2 +- pkg/query-service/model/response.go | 26 ----------------- pkg/query-service/model/trace.go | 29 +++++++++++++++++++ 3 files changed, 30 insertions(+), 27 deletions(-) create mode 100644 pkg/query-service/model/trace.go diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 0777cb3ebf..80e4f08a45 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -1134,7 +1134,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea } var startTime, endTime, durationNano uint64 - var searchScanResponses []model.SearchSpanResponseItemV2 + var searchScanResponses []model.SpanItemV2 query := fmt.Sprintf("SELECT timestamp, duration_nano, span_id, trace_id, has_error, kind, resource_string_service$$name, name, references, attributes_string, events, status_message, status_code_string, kind_string FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceTableName) diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 2d86dfa862..4d7fb9d230 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -269,32 +269,6 @@ type SearchSpanResponseItem struct { SpanKind string `json:"spanKind"` } -type SearchSpanResponseItemV2 struct { - TimeUnixNano time.Time `json:"timestamp" ch:"timestamp"` - DurationNano uint64 `json:"durationNano" ch:"duration_nano"` - SpanID string `json:"spanId" ch:"span_id"` - TraceID string `json:"traceId" ch:"trace_id"` - HasError bool `json:"hasError" ch:"has_error"` - Kind int8 `json:"kind" ch:"kind"` - ServiceName string `json:"serviceName" ch:"resource_string_service$$name"` - Name string `json:"name" ch:"name"` - References string `json:"references,omitempty" ch:"references"` - Attributes_string map[string]string `json:"attributes_string" ch:"attributes_string"` - Attributes_number map[string]float64 `json:"attributes_number" ch:"attributes_number"` - Attributes_bool map[string]bool `json:"attributes_bool" ch:"attributes_bool"` - Events []string `json:"event" ch:"events"` - StatusMessage string `json:"statusMessage" ch:"status_message"` - StatusCodeString string `json:"statusCodeString" ch:"status_code_string"` - SpanKind string `json:"spanKind" ch:"kind_string"` -} - -type TraceSummary struct { - TraceID string `json:"traceId" ch:"trace_id"` - Start time.Time `json:"start" ch:"start"` - End time.Time `json:"end" ch:"end"` - NumSpans uint64 `json:"numSpans" ch:"num_spans"` -} - type OtelSpanRef struct { TraceId string `json:"traceId,omitempty"` SpanId string `json:"spanId,omitempty"` diff --git a/pkg/query-service/model/trace.go b/pkg/query-service/model/trace.go new file mode 100644 index 0000000000..d334e1cfe7 --- /dev/null +++ b/pkg/query-service/model/trace.go @@ -0,0 +1,29 @@ +package model + +import "time" + +type SpanItemV2 struct { + TimeUnixNano time.Time `ch:"timestamp"` + DurationNano uint64 `ch:"duration_nano"` + SpanID string `ch:"span_id"` + TraceID string `ch:"trace_id"` + HasError bool `ch:"has_error"` + Kind int8 `ch:"kind"` + ServiceName string `ch:"resource_string_service$$name"` + Name string `ch:"name"` + References string `ch:"references"` + Attributes_string map[string]string `ch:"attributes_string"` + Attributes_number map[string]float64 `ch:"attributes_number"` + Attributes_bool map[string]bool `ch:"attributes_bool"` + Events []string `ch:"events"` + StatusMessage string `ch:"status_message"` + StatusCodeString string `ch:"status_code_string"` + SpanKind string `ch:"kind_string"` +} + +type TraceSummary struct { + TraceID string `ch:"trace_id"` + Start time.Time `ch:"start"` + End time.Time `ch:"end"` + NumSpans uint64 `ch:"num_spans"` +} From 60f13b3eea3dd7e52da8b38409d9239f2856bdf7 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Tue, 19 Nov 2024 17:05:40 +0530 Subject: [PATCH 36/41] fix: minor fixes --- pkg/query-service/app/clickhouseReader/reader.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 80e4f08a45..62cbad1cf5 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -490,7 +490,7 @@ func (r *ClickHouseReader) GetServicesList(ctx context.Context) (*[]string, erro query := fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceTableName) if r.useTraceNewSchema { - query = fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE ts_bucket_start > (toUnixTimestamp(now()) - 1800) AND toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceTableName) + query = fmt.Sprintf(`SELECT DISTINCT serviceName FROM %s.%s WHERE ts_bucket_start > (toUnixTimestamp(now() - INTERVAL 1 DAY) - 1800) AND toDate(timestamp) > now() - INTERVAL 1 DAY`, r.TraceDB, r.traceTableName) } rows, err := r.db.Query(ctx, query) @@ -3978,7 +3978,7 @@ func (r *ClickHouseReader) GetTraceAggregateAttributesV2(ctx context.Context, re } } - // add other attributes + // add the new static fields for _, field := range constants.NewStaticFieldsTraces { if (!stringAllowed && field.DataType == v3.AttributeKeyDataTypeString) || (v3.AttributeKey{} == field) { continue @@ -4101,12 +4101,14 @@ func (r *ClickHouseReader) GetTraceAttributeKeysV2(ctx context.Context, req *v3. } // don't send deprecated static fields + // this is added so that once the old tenants are moved to new schema, + // they old attributes are not sent to the frontend autocomplete if _, ok := constants.DeprecatedStaticFieldsTraces[tagKey]; !ok { response.AttributeKeys = append(response.AttributeKeys, key) } } - // add other attributes + // add the new static fields for _, f := range constants.NewStaticFieldsTraces { if (v3.AttributeKey{} == f) { continue From 78cf7757bf635fd188a8f412060ca502b63ed830 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 20 Nov 2024 16:46:58 +0530 Subject: [PATCH 37/41] fix: address comments --- .../app/clickhouseReader/reader.go | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 62cbad1cf5..ff1024e541 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -619,12 +619,26 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G ) if r.useTraceNewSchema { - bFilter := " AND ts_bucket_start >= @start_bucket AND ts_bucket_start <= @end_bucket" - query += bFilter - errorQuery += bFilter + resourceBucketFilter := ` + AND ( + resource_fingerprint GLOBAL IN + ( + SELECT fingerprint FROM %s.%s + WHERE + seen_at_ts_bucket_start >= @start_bucket AND seen_at_ts_bucket_start <= @end_bucket AND + simpleJSONExtractString(labels, 'service.name') = @serviceName AND + labels like @labelFilter + ) + ) + AND ts_bucket_start >= @start_bucket AND ts_bucket_start <= @end_bucket + ` + resourceBucketFilter = fmt.Sprintf(resourceBucketFilter, r.TraceDB, r.traceResourceTableV3) + query += resourceBucketFilter + errorQuery += resourceBucketFilter args = append(args, clickhouse.Named("start_bucket", strconv.FormatInt(queryParams.Start.Unix()-1800, 10)), clickhouse.Named("end_bucket", strconv.FormatInt(queryParams.End.Unix(), 10)), + clickhouse.Named("labelFilter", "%service.name%"+strings.ToLower(utils.QuoteEscapedStringForContains(svc, true))+"%"), ) } @@ -1026,6 +1040,30 @@ func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *mo WHERE serviceName = @serviceName AND timestamp>= @start AND timestamp<= @end`, r.TraceDB, r.traceTableName, ) + + if r.useTraceNewSchema { + resourceBucketFilter := ` + AND ( + resource_fingerprint GLOBAL IN + ( + SELECT fingerprint FROM %s.%s + WHERE + seen_at_ts_bucket_start >= @start_bucket AND seen_at_ts_bucket_start <= @end_bucket AND + simpleJSONExtractString(labels, 'service.name') = @serviceName AND + labels like @labelFilter + ) + ) + AND ts_bucket_start >= @start_bucket AND ts_bucket_start <= @end_bucket + ` + resourceBucketFilter = fmt.Sprintf(resourceBucketFilter, r.TraceDB, r.traceResourceTableV3) + query += resourceBucketFilter + namedArgs = append(namedArgs, + clickhouse.Named("start_bucket", strconv.FormatInt(queryParams.Start.Unix()-1800, 10)), + clickhouse.Named("end_bucket", strconv.FormatInt(queryParams.End.Unix(), 10)), + clickhouse.Named("labelFilter", "%service.name%"+strings.ToLower(utils.QuoteEscapedStringForContains(queryParams.ServiceName, true))+"%"), + ) + } + args := []interface{}{} args = append(args, namedArgs...) // create TagQuery from TagQueryParams @@ -1136,7 +1174,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea var startTime, endTime, durationNano uint64 var searchScanResponses []model.SpanItemV2 - query := fmt.Sprintf("SELECT timestamp, duration_nano, span_id, trace_id, has_error, kind, resource_string_service$$name, name, references, attributes_string, events, status_message, status_code_string, kind_string FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceTableName) + query := fmt.Sprintf("SELECT timestamp, duration_nano, span_id, trace_id, has_error, kind, resource_string_service$$name, name, references, attributes_string, attributes_number, attributes_bool, events, status_message, status_code_string, kind_string FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceTableName) start := time.Now() From 9f25cca0d20e2117b9e8cb6c87f001377825bf29 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 20 Nov 2024 17:00:52 +0530 Subject: [PATCH 38/41] fix: add to serviceoverview function --- .../app/clickhouseReader/reader.go | 39 ++++++------------- pkg/query-service/constants/constants.go | 14 +++++++ 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index ff1024e541..435f5a7aea 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -619,20 +619,7 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G ) if r.useTraceNewSchema { - resourceBucketFilter := ` - AND ( - resource_fingerprint GLOBAL IN - ( - SELECT fingerprint FROM %s.%s - WHERE - seen_at_ts_bucket_start >= @start_bucket AND seen_at_ts_bucket_start <= @end_bucket AND - simpleJSONExtractString(labels, 'service.name') = @serviceName AND - labels like @labelFilter - ) - ) - AND ts_bucket_start >= @start_bucket AND ts_bucket_start <= @end_bucket - ` - resourceBucketFilter = fmt.Sprintf(resourceBucketFilter, r.TraceDB, r.traceResourceTableV3) + resourceBucketFilter := fmt.Sprintf(constants.TraceResourceBucketFilterWithServiceName, r.TraceDB, r.traceResourceTableV3) query += resourceBucketFilter errorQuery += resourceBucketFilter args = append(args, @@ -726,6 +713,15 @@ func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams * WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end`, r.TraceDB, r.traceTableName, ) + if r.useTraceNewSchema { + resourceBucketFilter := fmt.Sprintf(constants.TraceResourceBucketFilterWithServiceName, r.TraceDB, r.traceResourceTableV3) + query += resourceBucketFilter + namedArgs = append(namedArgs, + clickhouse.Named("start_bucket", strconv.FormatInt(queryParams.Start.Unix()-1800, 10)), + clickhouse.Named("end_bucket", strconv.FormatInt(queryParams.End.Unix(), 10)), + clickhouse.Named("labelFilter", "%service.name%"+strings.ToLower(utils.QuoteEscapedStringForContains(queryParams.ServiceName, true))+"%"), + ) + } args := []interface{}{} args = append(args, namedArgs...) @@ -1042,20 +1038,7 @@ func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *mo ) if r.useTraceNewSchema { - resourceBucketFilter := ` - AND ( - resource_fingerprint GLOBAL IN - ( - SELECT fingerprint FROM %s.%s - WHERE - seen_at_ts_bucket_start >= @start_bucket AND seen_at_ts_bucket_start <= @end_bucket AND - simpleJSONExtractString(labels, 'service.name') = @serviceName AND - labels like @labelFilter - ) - ) - AND ts_bucket_start >= @start_bucket AND ts_bucket_start <= @end_bucket - ` - resourceBucketFilter = fmt.Sprintf(resourceBucketFilter, r.TraceDB, r.traceResourceTableV3) + resourceBucketFilter := fmt.Sprintf(constants.TraceResourceBucketFilterWithServiceName, r.TraceDB, r.traceResourceTableV3) query += resourceBucketFilter namedArgs = append(namedArgs, clickhouse.Named("start_bucket", strconv.FormatInt(queryParams.Start.Unix()-1800, 10)), diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 7072f13173..b66f60dfa9 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -718,3 +718,17 @@ func init() { } const TRACE_V4_MAX_PAGINATION_LIMIT = 10000 + +const TraceResourceBucketFilterWithServiceName = ` + AND ( + resource_fingerprint GLOBAL IN + ( + SELECT fingerprint FROM %s.%s + WHERE + seen_at_ts_bucket_start >= @start_bucket AND seen_at_ts_bucket_start <= @end_bucket AND + simpleJSONExtractString(labels, 'service.name') = @serviceName AND + labels like @labelFilter + ) + ) + AND ts_bucket_start >= @start_bucket AND ts_bucket_start <= @end_bucket + ` From 434a6c93caade494d838bd0a6150347b46164273 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 20 Nov 2024 17:10:53 +0530 Subject: [PATCH 39/41] fix: add changes to overview function --- pkg/query-service/app/clickhouseReader/reader.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 435f5a7aea..86ccd064a8 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -753,6 +753,11 @@ func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams * WHERE serviceName = @serviceName AND name In @names AND timestamp>= @start AND timestamp<= @end AND statusCode=2`, r.TraceDB, r.traceTableName, ) + if r.useTraceNewSchema { + resourceBucketFilter := fmt.Sprintf(constants.TraceResourceBucketFilterWithServiceName, r.TraceDB, r.traceResourceTableV3) + query += resourceBucketFilter + // args are already added in namedArgs + } args = []interface{}{} args = append(args, namedArgs...) subQuery, argsSubQuery, errStatus = buildQueryWithTagParams(ctx, tags) From 5e8350e5dc44d0c063fc7fce116d6a183be9d73e Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Wed, 20 Nov 2024 23:00:17 +0530 Subject: [PATCH 40/41] fix: address comments --- pkg/query-service/app/clickhouseReader/reader.go | 6 +++++- pkg/query-service/model/trace.go | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 86ccd064a8..43df012b2f 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -1162,7 +1162,7 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea var startTime, endTime, durationNano uint64 var searchScanResponses []model.SpanItemV2 - query := fmt.Sprintf("SELECT timestamp, duration_nano, span_id, trace_id, has_error, kind, resource_string_service$$name, name, references, attributes_string, attributes_number, attributes_bool, events, status_message, status_code_string, kind_string FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceTableName) + query := fmt.Sprintf("SELECT timestamp, duration_nano, span_id, trace_id, has_error, kind, resource_string_service$$name, name, references, attributes_string, attributes_number, attributes_bool, resources_string, events, status_message, status_code_string, kind_string FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3", r.TraceDB, r.traceTableName) start := time.Now() @@ -1196,6 +1196,9 @@ func (r *ClickHouseReader) SearchTracesV2(ctx context.Context, params *model.Sea for k, v := range item.Attributes_number { item.Attributes_string[k] = fmt.Sprintf("%v", v) } + for k, v := range item.Resources_string { + item.Attributes_string[k] = v + } jsonItem := model.SearchSpanResponseItem{ SpanID: item.SpanID, @@ -4234,6 +4237,7 @@ func (r *ClickHouseReader) GetTraceAttributeValuesV2(ctx context.Context, req *v selectKey = fmt.Sprintf("toInt64(%s)", req.FilterAttributeKey) } + // TODO(nitya): remove 24 hour limit in future after checking the perf/resource implications query = fmt.Sprintf("select distinct %s from %s.%s where ts_bucket_start >= toUInt64(toUnixTimestamp(now() - INTERVAL 48 HOUR)) AND timestamp >= toDateTime64(now() - INTERVAL 48 HOUR, 9) and %s ILIKE $1 limit $2", selectKey, r.TraceDB, r.traceTableName, filterValueColumnWhere) rows, err = r.db.Query(ctx, query, searchText, req.Limit) } else { diff --git a/pkg/query-service/model/trace.go b/pkg/query-service/model/trace.go index d334e1cfe7..e8d3d70ac2 100644 --- a/pkg/query-service/model/trace.go +++ b/pkg/query-service/model/trace.go @@ -15,6 +15,7 @@ type SpanItemV2 struct { Attributes_string map[string]string `ch:"attributes_string"` Attributes_number map[string]float64 `ch:"attributes_number"` Attributes_bool map[string]bool `ch:"attributes_bool"` + Resources_string map[string]string `ch:"resources_string"` Events []string `ch:"events"` StatusMessage string `ch:"status_message"` StatusCodeString string `ch:"status_code_string"` From c31e9cdda65e9b595e2653504c8bb701cd54d835 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Thu, 21 Nov 2024 11:43:34 +0530 Subject: [PATCH 41/41] fix: remove printlines --- pkg/query-service/rules/threshold_rule_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/query-service/rules/threshold_rule_test.go b/pkg/query-service/rules/threshold_rule_test.go index 2582ae56fc..5a4ab9c970 100644 --- a/pkg/query-service/rules/threshold_rule_test.go +++ b/pkg/query-service/rules/threshold_rule_test.go @@ -2,7 +2,6 @@ package rules import ( "context" - "fmt" "strings" "testing" "time" @@ -1120,8 +1119,6 @@ type queryMatcherAny struct { } func (m *queryMatcherAny) Match(x string, y string) error { - fmt.Println(x) - fmt.Println(y) return nil }