From 2b0da82f944005e21aec7a317a2d3a6bd14af615 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Tue, 22 Oct 2024 16:46:58 +0530 Subject: [PATCH] feat: move resource qb to its own package and use common options (#6238) * feat: move resource qb to its own package and use common options * fix: add remaining files * fix: remove redundant struct --- .../app/logs/v3/query_builder.go | 2 +- .../app/logs/v3/query_builder_test.go | 24 ++++++++--------- .../app/logs/v4/query_builder.go | 8 +++--- .../app/logs/v4/query_builder_test.go | 12 ++++----- pkg/query-service/app/querier/helper.go | 12 ++++----- pkg/query-service/app/querier/v2/helper.go | 12 ++++----- .../app/queryBuilder/query_builder.go | 19 +++++++------- .../v4 => resource}/resource_query_builder.go | 8 +++--- .../resource_query_builder_test.go | 4 +-- .../app/traces/v3/query_builder.go | 9 ++----- .../app/traces/v3/query_builder_test.go | 26 +++++++++---------- pkg/query-service/model/v3/v3.go | 2 +- 12 files changed, 67 insertions(+), 71 deletions(-) rename pkg/query-service/app/{logs/v4 => resource}/resource_query_builder.go (95%) rename pkg/query-service/app/{logs/v4 => resource}/resource_query_builder_test.go (98%) diff --git a/pkg/query-service/app/logs/v3/query_builder.go b/pkg/query-service/app/logs/v3/query_builder.go index 05bd799712..40f019ac91 100644 --- a/pkg/query-service/app/logs/v3/query_builder.go +++ b/pkg/query-service/app/logs/v3/query_builder.go @@ -496,7 +496,7 @@ func IsOrderByTs(orderBy []v3.OrderBy) bool { // PrepareLogsQuery prepares the query for logs // start and end are in epoch millisecond // step is in seconds -func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.LogQBOptions) (string, error) { +func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.QBOptions) (string, error) { // adjust the start and end time to the step interval // NOTE: Disabling this as it's creating confusion between charts and actual data diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index 0eb0c202e5..868afd0015 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -1201,7 +1201,7 @@ var testPrepLogsQueryData = []struct { TableName string AggregateOperator v3.AggregateOperator ExpectedQuery string - Options v3.LogQBOptions + Options v3.QBOptions }{ { Name: "Test TS with limit- first", @@ -1223,7 +1223,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT `method` from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') group by `method` order by value DESC) LIMIT 10", - Options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- first - with order by value", @@ -1246,7 +1246,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT `method` from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') group by `method` order by value ASC) LIMIT 10", - Options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- first - with order by attribute", @@ -1269,7 +1269,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT `method` from (SELECT attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') group by `method` order by `method` ASC) LIMIT 10", - Options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, { Name: "Test TS with limit- second", @@ -1291,7 +1291,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') AND (`method`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `method`,ts order by value DESC", - Options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, + Options: v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, { Name: "Test TS with limit- second - with order by", @@ -1314,7 +1314,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string_value[indexOf(attributes_string_key, 'method')] as `method`, toFloat64(count(distinct(attributes_string_value[indexOf(attributes_string_key, 'name')]))) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND has(attributes_string_key, 'method') AND has(attributes_string_key, 'name') AND (`method`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `method`,ts order by `method` ASC", - Options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, + Options: v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, // Live tail { @@ -1334,7 +1334,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where attributes_string_value[indexOf(attributes_string_key, 'method')] = 'GET' AND ", - Options: v3.LogQBOptions{IsLivetailQuery: true}, + Options: v3.QBOptions{IsLivetailQuery: true}, }, { Name: "Live Tail Query with contains", @@ -1353,7 +1353,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where attributes_string_value[indexOf(attributes_string_key, 'method')] ILIKE '%GET%' AND ", - Options: v3.LogQBOptions{IsLivetailQuery: true}, + Options: v3.QBOptions{IsLivetailQuery: true}, }, { Name: "Live Tail Query W/O filter", @@ -1369,7 +1369,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where ", - Options: v3.LogQBOptions{IsLivetailQuery: true}, + Options: v3.QBOptions{IsLivetailQuery: true}, }, { Name: "Table query w/o limit", @@ -1385,7 +1385,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) order by value DESC", - Options: v3.LogQBOptions{}, + Options: v3.QBOptions{}, }, { Name: "Table query with limit", @@ -1402,7 +1402,7 @@ var testPrepLogsQueryData = []struct { }, TableName: "logs", ExpectedQuery: "SELECT now() as ts, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) order by value DESC LIMIT 10", - Options: v3.LogQBOptions{}, + Options: v3.QBOptions{}, }, { Name: "Ignore offset if order by is timestamp in list queries", @@ -1488,7 +1488,7 @@ var testPrepLogsQueryLimitOffsetData = []struct { TableName string AggregateOperator v3.AggregateOperator ExpectedQuery string - Options v3.LogQBOptions + Options v3.QBOptions }{ { Name: "Test limit less than pageSize - order by ts", diff --git a/pkg/query-service/app/logs/v4/query_builder.go b/pkg/query-service/app/logs/v4/query_builder.go index 49e585e64b..e38fb94934 100644 --- a/pkg/query-service/app/logs/v4/query_builder.go +++ b/pkg/query-service/app/logs/v4/query_builder.go @@ -5,6 +5,7 @@ import ( "strings" logsV3 "go.signoz.io/signoz/pkg/query-service/app/logs/v3" + "go.signoz.io/signoz/pkg/query-service/app/resource" "go.signoz.io/signoz/pkg/query-service/constants" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" "go.signoz.io/signoz/pkg/query-service/utils" @@ -33,6 +34,7 @@ const ( BODY = "body" DISTRIBUTED_LOGS_V2 = "distributed_logs_v2" DISTRIBUTED_LOGS_V2_RESOURCE = "distributed_logs_v2_resource" + DB_NAME = "signoz_logs" NANOSECOND = 1000000000 ) @@ -372,7 +374,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build } // build the where clause for resource table - resourceSubQuery, err := buildResourceSubQuery(bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) + resourceSubQuery, err := resource.BuildResourceSubQuery(DB_NAME, DISTRIBUTED_LOGS_V2_RESOURCE, bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false) if err != nil { return "", err } @@ -463,7 +465,7 @@ func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) { } // no values for bucket start and end - resourceSubQuery, err := buildResourceSubQuery(0, 0, mq.Filters, mq.GroupBy, mq.AggregateAttribute, true) + resourceSubQuery, err := resource.BuildResourceSubQuery(DB_NAME, DISTRIBUTED_LOGS_V2_RESOURCE, 0, 0, mq.Filters, mq.GroupBy, mq.AggregateAttribute, true) if err != nil { return "", err } @@ -491,7 +493,7 @@ func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) { } // PrepareLogsQuery prepares the query for logs -func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.LogQBOptions) (string, error) { +func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.QBOptions) (string, error) { // adjust the start and end time to the step interval // NOTE: Disabling this as it's creating confusion between charts and actual data diff --git a/pkg/query-service/app/logs/v4/query_builder_test.go b/pkg/query-service/app/logs/v4/query_builder_test.go index 9c2b1fd2e5..64ba5c5004 100644 --- a/pkg/query-service/app/logs/v4/query_builder_test.go +++ b/pkg/query-service/app/logs/v4/query_builder_test.go @@ -806,7 +806,7 @@ func TestPrepareLogsQuery(t *testing.T) { queryType v3.QueryType panelType v3.PanelType mq *v3.BuilderQuery - options v3.LogQBOptions + options v3.QBOptions } tests := []struct { name string @@ -875,7 +875,7 @@ func TestPrepareLogsQuery(t *testing.T) { Limit: 10, GroupBy: []v3.AttributeKey{{Key: "user", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, }, - options: v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, + options: v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: true}, }, want: "SELECT `user` from (SELECT attributes_string['user'] as `user`, toFloat64(count(distinct(attributes_string['name']))) as value from signoz_logs.distributed_logs_v2 " + "where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['method'] = 'GET' " + @@ -904,7 +904,7 @@ func TestPrepareLogsQuery(t *testing.T) { GroupBy: []v3.AttributeKey{{Key: "user", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, Limit: 2, }, - options: v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, + options: v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit}, }, want: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, attributes_string['user'] as `user`, toFloat64(count(distinct(attributes_string['name']))) as value " + "from signoz_logs.distributed_logs_v2 where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND " + @@ -929,7 +929,7 @@ func TestPrepareLogsQuery(t *testing.T) { }, }, }, - options: v3.LogQBOptions{IsLivetailQuery: true}, + options: v3.QBOptions{IsLivetailQuery: true}, }, want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string " + "from signoz_logs.distributed_logs_v2 where attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND ", @@ -952,7 +952,7 @@ func TestPrepareLogsQuery(t *testing.T) { }, }, }, - options: v3.LogQBOptions{IsLivetailQuery: true}, + options: v3.QBOptions{IsLivetailQuery: true}, }, want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string from " + "signoz_logs.distributed_logs_v2 where attributes_string['method'] = 'GET' AND mapContains(attributes_string, 'method') AND " + @@ -972,7 +972,7 @@ func TestPrepareLogsQuery(t *testing.T) { Expression: "A", Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}, }, - options: v3.LogQBOptions{IsLivetailQuery: true}, + options: v3.QBOptions{IsLivetailQuery: true}, }, want: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string " + "from signoz_logs.distributed_logs_v2 where ", diff --git a/pkg/query-service/app/querier/helper.go b/pkg/query-service/app/querier/helper.go index 00b287ce8e..798eb8f0b7 100644 --- a/pkg/query-service/app/querier/helper.go +++ b/pkg/query-service/app/querier/helper.go @@ -45,7 +45,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -56,7 +56,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -71,7 +71,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{PreferRPM: preferRPM}, + v3.QBOptions{PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -167,7 +167,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} @@ -178,7 +178,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} @@ -191,7 +191,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{PreferRPM: preferRPM}, + v3.QBOptions{PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: query, Series: nil} diff --git a/pkg/query-service/app/querier/v2/helper.go b/pkg/query-service/app/querier/v2/helper.go index bb41bc8c36..09d6cc2309 100644 --- a/pkg/query-service/app/querier/v2/helper.go +++ b/pkg/query-service/app/querier/v2/helper.go @@ -44,7 +44,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -55,7 +55,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -70,7 +70,7 @@ func prepareLogsQuery(_ context.Context, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, - v3.LogQBOptions{PreferRPM: preferRPM}, + v3.QBOptions{PreferRPM: preferRPM}, ) if err != nil { return query, err @@ -167,7 +167,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} @@ -178,7 +178,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, + v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil} @@ -191,7 +191,7 @@ func (q *querier) runBuilderQuery( end, params.CompositeQuery.PanelType, builderQuery, - tracesV3.Options{PreferRPM: preferRPM}, + v3.QBOptions{PreferRPM: preferRPM}, ) if err != nil { ch <- channelResult{Err: err, Name: queryName, Query: query, Series: nil} diff --git a/pkg/query-service/app/queryBuilder/query_builder.go b/pkg/query-service/app/queryBuilder/query_builder.go index 879c2d5153..de8db2057a 100644 --- a/pkg/query-service/app/queryBuilder/query_builder.go +++ b/pkg/query-service/app/queryBuilder/query_builder.go @@ -6,7 +6,6 @@ import ( "github.com/SigNoz/govaluate" metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3" - tracesV3 "go.signoz.io/signoz/pkg/query-service/app/traces/v3" "go.signoz.io/signoz/pkg/query-service/cache" "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/interfaces" @@ -42,8 +41,8 @@ var SupportedFunctions = []string{ var EvalFuncs = map[string]govaluate.ExpressionFunction{} -type prepareTracesQueryFunc func(start, end int64, panelType v3.PanelType, bq *v3.BuilderQuery, options tracesV3.Options) (string, error) -type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, options v3.LogQBOptions) (string, error) +type prepareTracesQueryFunc func(start, end int64, panelType v3.PanelType, bq *v3.BuilderQuery, options v3.QBOptions) (string, error) +type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, options v3.QBOptions) (string, error) type prepareMetricQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, options metricsV3.Options) (string, error) type QueryBuilder struct { @@ -161,7 +160,7 @@ func (qb *QueryBuilder) PrepareLiveTailQuery(params *v3.QueryRangeParamsV3) (str } for queryName, query := range compositeQuery.BuilderQueries { if query.Expression == queryName { - queryStr, err = qb.options.BuildLogQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{IsLivetailQuery: true}) + queryStr, err = qb.options.BuildLogQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.QBOptions{IsLivetailQuery: true}) if err != nil { return "", err } @@ -195,12 +194,12 @@ func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3) (map[strin // for ts query with group by and limit form two queries if compositeQuery.PanelType == v3.PanelTypeGraph && query.Limit > 0 && len(query.GroupBy) > 0 { limitQuery, err := qb.options.BuildTraceQuery(start, end, compositeQuery.PanelType, query, - tracesV3.Options{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } placeholderQuery, err := qb.options.BuildTraceQuery(start, end, compositeQuery.PanelType, - query, tracesV3.Options{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + query, v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } @@ -208,7 +207,7 @@ func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3) (map[strin queries[queryName] = query } else { queryString, err := qb.options.BuildTraceQuery(start, end, compositeQuery.PanelType, - query, tracesV3.Options{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) + query, v3.QBOptions{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) if err != nil { return nil, err } @@ -217,18 +216,18 @@ func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3) (map[strin case v3.DataSourceLogs: // for ts query with limit replace it as it is already formed if compositeQuery.PanelType == v3.PanelTypeGraph && query.Limit > 0 && len(query.GroupBy) > 0 { - limitQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + limitQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.QBOptions{GraphLimitQtype: constants.FirstQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } - placeholderQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) + placeholderQuery, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.QBOptions{GraphLimitQtype: constants.SecondQueryGraphLimit, PreferRPM: PreferRPMFeatureEnabled}) if err != nil { return nil, err } query := fmt.Sprintf(placeholderQuery, limitQuery) queries[queryName] = query } else { - queryString, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.LogQBOptions{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) + queryString, err := qb.options.BuildLogQuery(start, end, compositeQuery.QueryType, compositeQuery.PanelType, query, v3.QBOptions{PreferRPM: PreferRPMFeatureEnabled, GraphLimitQtype: ""}) if err != nil { return nil, err } diff --git a/pkg/query-service/app/logs/v4/resource_query_builder.go b/pkg/query-service/app/resource/resource_query_builder.go similarity index 95% rename from pkg/query-service/app/logs/v4/resource_query_builder.go rename to pkg/query-service/app/resource/resource_query_builder.go index 3f7f2682cb..bbf9310386 100644 --- a/pkg/query-service/app/logs/v4/resource_query_builder.go +++ b/pkg/query-service/app/resource/resource_query_builder.go @@ -1,4 +1,4 @@ -package v4 +package resource import ( "fmt" @@ -211,7 +211,7 @@ func buildResourceFiltersFromAggregateAttribute(aggregateAttribute v3.AttributeK return "" } -func buildResourceSubQuery(bucketStart, bucketEnd int64, fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey, isLiveTail bool) (string, error) { +func BuildResourceSubQuery(dbName, tableName string, bucketStart, bucketEnd int64, fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey, isLiveTail bool) (string, error) { // BUILD THE WHERE CLAUSE var conditions []string @@ -242,10 +242,10 @@ func buildResourceSubQuery(bucketStart, bucketEnd int64, fs *v3.FilterSet, group // BUILD THE FINAL QUERY var query string if isLiveTail { - query = fmt.Sprintf("SELECT fingerprint FROM signoz_logs.%s WHERE ", DISTRIBUTED_LOGS_V2_RESOURCE) + query = fmt.Sprintf("SELECT fingerprint FROM %s.%s WHERE ", dbName, tableName) query = "(" + query + conditionStr } else { - query = fmt.Sprintf("SELECT fingerprint FROM signoz_logs.%s WHERE (seen_at_ts_bucket_start >= %d) AND (seen_at_ts_bucket_start <= %d) AND ", DISTRIBUTED_LOGS_V2_RESOURCE, bucketStart, bucketEnd) + query = fmt.Sprintf("SELECT fingerprint FROM %s.%s WHERE (seen_at_ts_bucket_start >= %d) AND (seen_at_ts_bucket_start <= %d) AND ", dbName, tableName, bucketStart, bucketEnd) query = "(" + query + conditionStr + ")" } diff --git a/pkg/query-service/app/logs/v4/resource_query_builder_test.go b/pkg/query-service/app/resource/resource_query_builder_test.go similarity index 98% rename from pkg/query-service/app/logs/v4/resource_query_builder_test.go rename to pkg/query-service/app/resource/resource_query_builder_test.go index 7bea0bc291..f390ff9c6e 100644 --- a/pkg/query-service/app/logs/v4/resource_query_builder_test.go +++ b/pkg/query-service/app/resource/resource_query_builder_test.go @@ -1,4 +1,4 @@ -package v4 +package resource import ( "reflect" @@ -497,7 +497,7 @@ func Test_buildResourceSubQuery(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := buildResourceSubQuery(tt.args.bucketStart, tt.args.bucketEnd, tt.args.fs, tt.args.groupBy, tt.args.aggregateAttribute, false) + got, err := BuildResourceSubQuery("signoz_logs", "distributed_logs_v2_resource", tt.args.bucketStart, tt.args.bucketEnd, tt.args.fs, tt.args.groupBy, tt.args.aggregateAttribute, false) if (err != nil) != tt.wantErr { t.Errorf("buildResourceSubQuery() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index fcb682cfbb..ce9c54573c 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -10,11 +10,6 @@ import ( "go.signoz.io/signoz/pkg/query-service/utils" ) -type Options struct { - GraphLimitQtype string - PreferRPM bool -} - var aggregateOperatorToPercentile = map[v3.AggregateOperator]float64{ v3.AggregateOperatorP05: 0.05, v3.AggregateOperatorP10: 0.10, @@ -238,7 +233,7 @@ func handleEmptyValuesInGroupBy(groupBy []v3.AttributeKey) (string, error) { return "", nil } -func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, panelType v3.PanelType, options Options) (string, error) { +func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, _ string, panelType v3.PanelType, options v3.QBOptions) (string, error) { filterSubQuery, err := buildTracesFilterQuery(mq.Filters) if err != nil { @@ -504,7 +499,7 @@ func addOffsetToQuery(query string, offset uint64) string { // PrepareTracesQuery returns the query string for traces // start and end are in epoch millisecond // step is in seconds -func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) { +func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, options v3.QBOptions) (string, error) { // adjust the start and end time to the step interval start = start - (start % (mq.StepInterval * 1000)) end = end - (end % (mq.StepInterval * 1000)) diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 3903a895dd..23b73a134d 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -490,7 +490,7 @@ var testBuildTracesQueryData = []struct { AggregateOperator v3.AggregateOperator ExpectedQuery string PanelType v3.PanelType - Options Options + Options v3.QBOptions }{ { Name: "Test aggregate count on fixed column of float64 type", @@ -524,7 +524,7 @@ var testBuildTracesQueryData = []struct { " signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <=" + " '1680066458000000000') group by ts order by value DESC", PanelType: v3.PanelTypeGraph, - Options: Options{GraphLimitQtype: "", PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: "", PreferRPM: true}, }, { Name: "Test aggregate count on fixed column of float64 type with filter", @@ -867,7 +867,7 @@ var testBuildTracesQueryData = []struct { "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + " AND has(stringTagMap, 'method') group by `method`,ts order by `method` ASC", PanelType: v3.PanelTypeGraph, - Options: Options{GraphLimitQtype: "", + Options: v3.QBOptions{GraphLimitQtype: "", PreferRPM: false, }, }, @@ -892,7 +892,7 @@ var testBuildTracesQueryData = []struct { "AND has(stringTagMap, 'method') group by `method`,ts " + "order by `method` ASC", PanelType: v3.PanelTypeGraph, - Options: Options{GraphLimitQtype: "", PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: "", PreferRPM: true}, }, { Name: "Test aggregate RateSum without fixed column", @@ -916,7 +916,7 @@ var testBuildTracesQueryData = []struct { "AND has(stringTagMap, 'method') group by `method`,ts " + "order by `method` ASC", PanelType: v3.PanelTypeGraph, - Options: Options{GraphLimitQtype: "", PreferRPM: true}, + Options: v3.QBOptions{GraphLimitQtype: "", PreferRPM: true}, }, { Name: "Test aggregate with having clause", @@ -1207,7 +1207,7 @@ var testPrepTracesQueryData = []struct { BuilderQuery *v3.BuilderQuery ExpectedQuery string Keys map[string]v3.AttributeKey - Options Options + Options v3.QBOptions }{ { Name: "Test TS with limit- first", @@ -1232,7 +1232,7 @@ var testPrepTracesQueryData = []struct { " where (timestamp >= '1680066360000000000' AND timestamp <= '1680066420000000000') AND" + " stringTagMap['method'] = 'GET' AND has(stringTagMap, 'method') group by `method` order by value DESC) LIMIT 10", Keys: map[string]v3.AttributeKey{"name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, @@ -1261,7 +1261,7 @@ var testPrepTracesQueryData = []struct { " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') group by `method` order by value ASC) LIMIT 10", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, @@ -1287,7 +1287,7 @@ var testPrepTracesQueryData = []struct { " AND timestamp <= '1680066420000000000') " + "group by `serviceName` order by `serviceName` ASC) LIMIT 10", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, @@ -1317,7 +1317,7 @@ var testPrepTracesQueryData = []struct { " AND timestamp <= '1680066420000000000') AND has(stringTagMap, 'http.method') " + "group by `serviceName`,`http.method` order by `serviceName` ASC,value ASC) LIMIT 10", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.FirstQueryGraphLimit, }, }, @@ -1345,7 +1345,7 @@ var testPrepTracesQueryData = []struct { " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by value DESC", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, @@ -1373,7 +1373,7 @@ var testPrepTracesQueryData = []struct { " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360000000000'" + " AND timestamp <= '1680066420000000000') AND stringTagMap['method'] = 'GET' AND" + " has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by `method` ASC", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, @@ -1408,7 +1408,7 @@ var testPrepTracesQueryData = []struct { "AND (`method`,`name`) GLOBAL IN (%s) group by `method`,`name`,ts " + "order by `method` ASC,`name` ASC", Keys: map[string]v3.AttributeKey{}, - Options: Options{ + Options: v3.QBOptions{ GraphLimitQtype: constants.SecondQueryGraphLimit, }, }, diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index a58575cd08..550fc65f5f 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -1308,7 +1308,7 @@ type URLShareableOptions struct { SelectColumns []AttributeKey `json:"selectColumns"` } -type LogQBOptions struct { +type QBOptions struct { GraphLimitQtype string IsLivetailQuery bool PreferRPM bool