diff --git a/pkg/query-service/app/querier/querier.go b/pkg/query-service/app/querier/querier.go index b18e56bcb6..7284bddc9f 100644 --- a/pkg/query-service/app/querier/querier.go +++ b/pkg/query-service/app/querier/querier.go @@ -250,7 +250,7 @@ func (q *querier) runBuilderQueries(ctx context.Context, params *v3.QueryRangePa } if builderQuery.DataSource == v3.DataSourceTraces { - query, err := tracesV3.PrepareTracesQuery(params.Start, params.End, params.CompositeQuery.QueryType, params.CompositeQuery.PanelType, builderQuery, keys) + query, err := tracesV3.PrepareTracesQuery(params.Start, params.End, params.CompositeQuery.PanelType, builderQuery, keys, "") if err != nil { errQueriesByName[queryName] = err.Error() continue diff --git a/pkg/query-service/app/queryBuilder/query_builder.go b/pkg/query-service/app/queryBuilder/query_builder.go index 1d16c8d709..c3f8b0d4d0 100644 --- a/pkg/query-service/app/queryBuilder/query_builder.go +++ b/pkg/query-service/app/queryBuilder/query_builder.go @@ -39,7 +39,7 @@ var SupportedFunctions = []string{ var EvalFuncs = map[string]govaluate.ExpressionFunction{} -type prepareTracesQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error) +type prepareTracesQueryFunc func(start, end int64, panelType v3.PanelType, bq *v3.BuilderQuery, keys map[string]v3.AttributeKey, graphLimitQtype string) (string, error) type prepareLogsQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery, graphLimitQtype string) (string, error) type prepareMetricQueryFunc func(start, end int64, queryType v3.QueryType, panelType v3.PanelType, bq *v3.BuilderQuery) (string, error) @@ -147,11 +147,25 @@ func (qb *QueryBuilder) PrepareQueries(params *v3.QueryRangeParamsV3, args ...in if len(args) > 0 { keys = args[0].(map[string]v3.AttributeKey) } - queryString, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.QueryType, compositeQuery.PanelType, query, keys) - if err != nil { - return nil, err + // for ts query with group by and limit form two queries + if compositeQuery.PanelType == v3.PanelTypeGraph && query.Limit > 0 && len(query.GroupBy) > 0 { + limitQuery, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.PanelType, query, keys, constants.FirstQueryGraphLimit) + if err != nil { + return nil, err + } + placeholderQuery, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.PanelType, query, keys, constants.SecondQueryGraphLimit) + if err != nil { + return nil, err + } + query := fmt.Sprintf(placeholderQuery, limitQuery) + queries[queryName] = query + } else { + queryString, err := qb.options.BuildTraceQuery(params.Start, params.End, compositeQuery.PanelType, query, keys, "") + if err != nil { + return nil, err + } + queries[queryName] = queryString } - queries[queryName] = queryString case v3.DataSourceLogs: // for ts query with limit replace it as it is already formed if compositeQuery.PanelType == v3.PanelTypeGraph && query.Limit > 0 && len(query.GroupBy) > 0 { diff --git a/pkg/query-service/app/traces/v3/query_builder.go b/pkg/query-service/app/traces/v3/query_builder.go index e8142eeead..1f9dfefbd9 100644 --- a/pkg/query-service/app/traces/v3/query_builder.go +++ b/pkg/query-service/app/traces/v3/query_builder.go @@ -108,6 +108,18 @@ func getSelectLabels(aggregatorOperator v3.AggregateOperator, groupBy []v3.Attri return selectLabels } +func getSelectKeys(aggregatorOperator v3.AggregateOperator, groupBy []v3.AttributeKey) string { + var selectLabels []string + if aggregatorOperator == v3.AggregateOperatorNoOp { + return "" + } else { + for _, tag := range groupBy { + selectLabels = append(selectLabels, fmt.Sprintf("`%s`", tag.Key)) + } + } + return strings.Join(selectLabels, ",") +} + func getSelectColumns(sc []v3.AttributeKey, keys map[string]v3.AttributeKey) string { var columns []string for _, tag := range sc { @@ -219,7 +231,7 @@ func handleEmptyValuesInGroupBy(keys map[string]v3.AttributeKey, groupBy []v3.At return "", nil } -func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey, panelType v3.PanelType) (string, error) { +func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName string, keys map[string]v3.AttributeKey, panelType v3.PanelType, graphLimitQtype string) (string, error) { filterSubQuery, err := buildTracesFilterQuery(mq.Filters, keys) if err != nil { @@ -236,24 +248,27 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str } var queryTmpl string - - if panelType == v3.PanelTypeTable { + if graphLimitQtype == constants.FirstQueryGraphLimit { + queryTmpl = "SELECT" + } else if panelType == v3.PanelTypeTable { queryTmpl = - "SELECT now() as ts," + selectLabels + - " %s as value " + - "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + - " where " + spanIndexTableTimeFilter + "%s" + - "%s%s" + - "%s" + "SELECT now() as ts," } else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue { // Select the aggregate value for interval queryTmpl = - fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts,", step) + selectLabels + - " %s as value " + - "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + - " where " + spanIndexTableTimeFilter + "%s" + - "%s%s" + - "%s" + fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %d SECOND) AS ts,", step) + } + + queryTmpl = queryTmpl + selectLabels + + " %s as value " + + "from " + constants.SIGNOZ_TRACE_DBNAME + "." + constants.SIGNOZ_SPAN_INDEX_TABLENAME + + " where " + spanIndexTableTimeFilter + "%s" + + "%s%s" + + "%s" + + // we don't need value for first query + if graphLimitQtype == constants.FirstQueryGraphLimit { + queryTmpl = "SELECT " + getSelectKeys(mq.AggregateOperator, mq.GroupBy) + " from (" + queryTmpl + ")" } emptyValuesInGroupByFilter, err := handleEmptyValuesInGroupBy(keys, mq.GroupBy) @@ -262,7 +277,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str } filterSubQuery += emptyValuesInGroupByFilter - groupBy := groupByAttributeKeyTags(panelType, mq.GroupBy...) + groupBy := groupByAttributeKeyTags(panelType, graphLimitQtype, mq.GroupBy...) if groupBy != "" { groupBy = " group by " + groupBy } @@ -271,6 +286,11 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str if orderBy != "" { orderBy = " order by " + orderBy } + + if graphLimitQtype == constants.SecondQueryGraphLimit { + filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", getSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "%s)" + } + aggregationKey := "" if mq.AggregateAttribute.Key != "" { aggregationKey = getColumnName(mq.AggregateAttribute, keys) @@ -326,7 +346,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str var query string if panelType == v3.PanelTypeTrace { withSubQuery := fmt.Sprintf(constants.TracesExplorerViewSQLSelectWithSubQuery, constants.SIGNOZ_TRACE_DBNAME, constants.SIGNOZ_SPAN_INDEX_TABLENAME, spanIndexTableTimeFilter, filterSubQuery) - withSubQuery = addLimitToQuery(withSubQuery, mq.Limit, panelType) + withSubQuery = addLimitToQuery(withSubQuery, mq.Limit) if mq.Offset != 0 { withSubQuery = addOffsetToQuery(withSubQuery, mq.Offset) } @@ -367,84 +387,60 @@ func enrichOrderBy(items []v3.OrderBy, keys map[string]v3.AttributeKey) []v3.Ord // groupBy returns a string of comma separated tags for group by clause // `ts` is always added to the group by clause -func groupBy(panelType v3.PanelType, tags ...string) string { - if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue { +func groupBy(panelType v3.PanelType, graphLimitQtype string, tags ...string) string { + if (graphLimitQtype != constants.FirstQueryGraphLimit) && (panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue) { tags = append(tags, "ts") } return strings.Join(tags, ",") } -func groupByAttributeKeyTags(panelType v3.PanelType, tags ...v3.AttributeKey) string { +func groupByAttributeKeyTags(panelType v3.PanelType, graphLimitQtype string, tags ...v3.AttributeKey) string { groupTags := []string{} for _, tag := range tags { groupTags = append(groupTags, fmt.Sprintf("`%s`", tag.Key)) } - return groupBy(panelType, groupTags...) + return groupBy(panelType, graphLimitQtype, groupTags...) } // orderBy returns a string of comma separated tags for order by clause // if there are remaining items which are not present in tags they are also added // if the order is not specified, it defaults to ASC -func orderBy(panelType v3.PanelType, items []v3.OrderBy, tags []string, keys map[string]v3.AttributeKey) []string { +func orderBy(panelType v3.PanelType, items []v3.OrderBy, tagLookup map[string]struct{}, keys map[string]v3.AttributeKey) []string { var orderBy []string - // create a lookup - addedToOrderBy := map[string]bool{} - itemsLookup := map[string]v3.OrderBy{} - - for i := 0; i < len(items); i++ { - addedToOrderBy[items[i].ColumnName] = false - itemsLookup[items[i].ColumnName] = items[i] - } - - for _, tag := range tags { - if item, ok := itemsLookup[tag]; ok { - orderBy = append(orderBy, fmt.Sprintf("`%s` %s", item.ColumnName, item.Order)) - addedToOrderBy[item.ColumnName] = true - } else { - orderBy = append(orderBy, fmt.Sprintf("`%s` ASC", tag)) - } - } - - // users might want to order by value of aggregation for _, item := range items { if item.ColumnName == constants.SigNozOrderByValue { orderBy = append(orderBy, fmt.Sprintf("value %s", item.Order)) - addedToOrderBy[item.ColumnName] = true - } - } - - // add the remaining items - if panelType == v3.PanelTypeList { - for _, item := range items { - // since these are not present in tags we will have to select them correctly - // for list view there is no need to check if it was added since they wont be added yet but this is just for safety - if !addedToOrderBy[item.ColumnName] { - attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn} - name := getColumnName(attr, keys) - - if item.IsColumn { - orderBy = append(orderBy, fmt.Sprintf("`%s` %s", name, item.Order)) - } else { - orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) - } + } else if _, ok := tagLookup[item.ColumnName]; ok { + orderBy = append(orderBy, fmt.Sprintf("`%s` %s", item.ColumnName, item.Order)) + } else if panelType == v3.PanelTypeList { + attr := v3.AttributeKey{Key: item.ColumnName, DataType: item.DataType, Type: item.Type, IsColumn: item.IsColumn} + name := getColumnName(attr, keys) + if item.IsColumn { + orderBy = append(orderBy, fmt.Sprintf("`%s` %s", name, item.Order)) + } else { + orderBy = append(orderBy, fmt.Sprintf("%s %s", name, item.Order)) } } } + return orderBy } func orderByAttributeKeyTags(panelType v3.PanelType, items []v3.OrderBy, tags []v3.AttributeKey, keys map[string]v3.AttributeKey) string { - var groupTags []string - for _, tag := range tags { - groupTags = append(groupTags, tag.Key) + tagLookup := map[string]struct{}{} + for _, v := range tags { + tagLookup[v.Key] = struct{}{} } - orderByArray := orderBy(panelType, items, groupTags, keys) - if panelType == v3.PanelTypeList && len(orderByArray) == 0 { - orderByArray = append(orderByArray, constants.TIMESTAMP+" DESC") - } else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue { - orderByArray = append(orderByArray, "ts") + orderByArray := orderBy(panelType, items, tagLookup, keys) + + if len(orderByArray) == 0 { + if panelType == v3.PanelTypeList { + orderByArray = append(orderByArray, constants.TIMESTAMP+" DESC") + } else if panelType == v3.PanelTypeGraph { + orderByArray = append(orderByArray, "value DESC") + } } str := strings.Join(orderByArray, ",") @@ -480,7 +476,7 @@ func reduceToQuery(query string, reduceTo v3.ReduceToOperator, aggregateOperator return query, nil } -func addLimitToQuery(query string, limit uint64, panelType v3.PanelType) string { +func addLimitToQuery(query string, limit uint64) string { if limit == 0 { limit = 100 } @@ -491,16 +487,33 @@ func addOffsetToQuery(query string, offset uint64) string { return fmt.Sprintf("%s OFFSET %d", query, offset) } -func PrepareTracesQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, keys map[string]v3.AttributeKey) (string, error) { - query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType) +func PrepareTracesQuery(start, end int64, panelType v3.PanelType, mq *v3.BuilderQuery, keys map[string]v3.AttributeKey, graphLimitQtype string) (string, error) { + if graphLimitQtype == constants.FirstQueryGraphLimit { + // give me just the group by names + query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType, graphLimitQtype) + if err != nil { + return "", err + } + query = addLimitToQuery(query, mq.Limit) + + return query, nil + } else if graphLimitQtype == constants.SecondQueryGraphLimit { + query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType, graphLimitQtype) + if err != nil { + return "", err + } + return query, nil + } + + query, err := buildTracesQuery(start, end, mq.StepInterval, mq, constants.SIGNOZ_SPAN_INDEX_TABLENAME, keys, panelType, graphLimitQtype) if err != nil { return "", err } if panelType == v3.PanelTypeValue { query, err = reduceToQuery(query, mq.ReduceTo, mq.AggregateOperator) } - if panelType == v3.PanelTypeList { - query = addLimitToQuery(query, mq.Limit, panelType) + if panelType == v3.PanelTypeList || panelType == v3.PanelTypeTable { + query = addLimitToQuery(query, mq.Limit) if mq.Offset != 0 { query = addOffsetToQuery(query, mq.Offset) diff --git a/pkg/query-service/app/traces/v3/query_builder_test.go b/pkg/query-service/app/traces/v3/query_builder_test.go index 89206b857d..c7abe75492 100644 --- a/pkg/query-service/app/traces/v3/query_builder_test.go +++ b/pkg/query-service/app/traces/v3/query_builder_test.go @@ -323,8 +323,8 @@ var testOrderBy = []struct { Name string PanelType v3.PanelType Items []v3.OrderBy - Tags []string - Result []string + Tags []v3.AttributeKey + Result string }{ { Name: "Test 1", @@ -339,8 +339,10 @@ var testOrderBy = []struct { Order: "desc", }, }, - Tags: []string{"name"}, - Result: []string{"`name` asc", "value desc"}, + Tags: []v3.AttributeKey{ + {Key: "name"}, + }, + Result: "`name` asc,value desc", }, { Name: "Test 2", @@ -355,8 +357,11 @@ var testOrderBy = []struct { Order: "asc", }, }, - Tags: []string{"name", "bytes"}, - Result: []string{"`name` asc", "`bytes` asc"}, + Tags: []v3.AttributeKey{ + {Key: "name"}, + {Key: "bytes"}, + }, + Result: "`name` asc,`bytes` asc", }, { Name: "Test 3", @@ -375,8 +380,11 @@ var testOrderBy = []struct { Order: "asc", }, }, - Tags: []string{"name", "bytes"}, - Result: []string{"`name` asc", "`bytes` asc", "value asc"}, + Tags: []v3.AttributeKey{ + {Key: "name"}, + {Key: "bytes"}, + }, + Result: "`name` asc,value asc,`bytes` asc", }, { Name: "Test 4", @@ -398,8 +406,11 @@ var testOrderBy = []struct { DataType: v3.AttributeKeyDataTypeString, }, }, - Tags: []string{"name", "bytes"}, - Result: []string{"`name` asc", "`bytes` asc", "stringTagMap['response_time'] desc"}, + Tags: []v3.AttributeKey{ + {Key: "name"}, + {Key: "bytes"}, + }, + Result: "`name` asc,`bytes` asc,stringTagMap['response_time'] desc", }, { Name: "Test 5", @@ -426,15 +437,15 @@ var testOrderBy = []struct { Order: "desc", }, }, - Tags: []string{}, - Result: []string{"`name` asc", "`bytes` asc", "stringTagMap['response_time'] desc"}, + Tags: []v3.AttributeKey{}, + Result: "`name` asc,`bytes` asc,stringTagMap['response_time'] desc", }, } func TestOrderBy(t *testing.T) { for _, tt := range testOrderBy { Convey("testOrderBy", t, func() { - res := orderBy(tt.PanelType, tt.Items, tt.Tags, map[string]v3.AttributeKey{ + res := orderByAttributeKeyTags(tt.PanelType, tt.Items, tt.Tags, map[string]v3.AttributeKey{ "name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, "bytes": {Key: "bytes", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, "response_time": {Key: "response_time", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: false}, @@ -470,7 +481,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " group by ts order by ts", + " group by ts order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -486,7 +497,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, count()/60 as value from" + " signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <=" + - " '1680066458000000000') group by ts order by ts", + " '1680066458000000000') group by ts order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -505,7 +516,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," + " toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v2" + " where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " AND stringTagMap['customer_id'] = '10001' group by ts order by ts", + " AND stringTagMap['customer_id'] = '10001' group by ts order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -522,7 +533,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " group by ts order by ts", + " group by ts order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -539,7 +550,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " AND has(stringTagMap, 'user_name') group by ts order by ts", + " AND has(stringTagMap, 'user_name') group by ts order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -556,7 +567,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " AND name != '' group by ts order by ts", + " AND name != '' group by ts order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -576,7 +587,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " AND numberTagMap['bytes'] > 100.000000 AND has(stringTagMap, 'user_name') group by ts order by ts", + " AND numberTagMap['bytes'] > 100.000000 AND has(stringTagMap, 'user_name') group by ts order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -594,7 +605,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(name))) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " group by ts order by value ASC,ts", + " group by ts order by value ASC", PanelType: v3.PanelTypeGraph, }, { @@ -611,7 +622,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name'])))" + " as value from signoz_traces.distributed_signoz_index_v2 where" + - " (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by ts", + " (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') group by ts order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -630,7 +641,7 @@ var testBuildTracesQueryData = []struct { }, }, GroupBy: []v3.AttributeKey{{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, - OrderBy: []v3.OrderBy{{ColumnName: "http.method", Order: "ASC"}, {ColumnName: "ts", Order: "ASC"}}, + OrderBy: []v3.OrderBy{{ColumnName: "http.method", Order: "ASC"}}, }, TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts," + @@ -639,7 +650,7 @@ var testBuildTracesQueryData = []struct { "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['http.method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + "AND has(stringTagMap, 'http.method') group by `http.method`,ts " + - "order by `http.method` ASC,ts", + "order by `http.method` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -671,7 +682,7 @@ var testBuildTracesQueryData = []struct { "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' AND resourceTagsMap['x'] != 'abc' " + "AND has(stringTagMap, 'method') AND has(resourceTagsMap, 'x') group by `method`,`x`,ts " + - "order by `method` ASC,`x` ASC,ts", + "order by `method` ASC,`x` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -699,7 +710,7 @@ var testBuildTracesQueryData = []struct { "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + "AND has(stringTagMap, 'method') group by `method`,ts " + - "order by `method` ASC,ts", + "order by `method` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -727,7 +738,7 @@ var testBuildTracesQueryData = []struct { "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + "AND has(stringTagMap, 'method') group by `method`,ts " + - "order by `method` ASC,ts", + "order by `method` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -755,7 +766,7 @@ var testBuildTracesQueryData = []struct { "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + "AND has(stringTagMap, 'method') group by `method`,ts " + - "order by `method` ASC,ts", + "order by `method` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -783,7 +794,7 @@ var testBuildTracesQueryData = []struct { "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND stringTagMap['method'] = 'GET' " + "AND has(stringTagMap, 'method') group by `method`,ts " + - "order by `method` ASC,ts", + "order by `method` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -807,7 +818,7 @@ var testBuildTracesQueryData = []struct { "from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND has(stringTagMap, 'method') group by `method`,ts " + - "order by `method` ASC,ts", + "order by `method` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -828,7 +839,7 @@ var testBuildTracesQueryData = []struct { ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, stringTagMap['method'] as `method`" + ", sum(bytes)/60 as value from signoz_traces.distributed_signoz_index_v2 " + "where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " AND has(stringTagMap, 'method') group by `method`,ts order by `method` ASC,ts", + " AND has(stringTagMap, 'method') group by `method`,ts order by `method` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -850,7 +861,7 @@ var testBuildTracesQueryData = []struct { ", count(numberTagMap['bytes'])/60 as value " + "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND has(stringTagMap, 'method') group by `method`,ts " + - "order by `method` ASC,ts", + "order by `method` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -873,7 +884,7 @@ var testBuildTracesQueryData = []struct { "sum(numberTagMap['bytes'])/60 as value " + "from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + "AND has(stringTagMap, 'method') group by `method`,ts " + - "order by `method` ASC,ts", + "order by `method` ASC", PanelType: v3.PanelTypeGraph, }, { @@ -897,7 +908,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')" + - " group by ts having value > 10 order by ts", + " group by ts having value > 10 order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -925,7 +936,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value from " + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by ts", + "AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -953,7 +964,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count(distinct(stringTagMap['name']))) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by ts", + "AND stringTagMap['method'] = 'GET' group by ts having value > 10 order by value DESC", PanelType: v3.PanelTypeGraph, }, { @@ -981,7 +992,7 @@ var testBuildTracesQueryData = []struct { TableName: "signoz_traces.distributed_signoz_index_v2", ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toFloat64(count()) as value" + " from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " + - "AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10 order by ts", + "AND stringTagMap['method'] = 'GET' AND has(stringTagMap, 'name') group by ts having value > 10", PanelType: v3.PanelTypeValue, }, { @@ -1120,7 +1131,218 @@ func TestBuildTracesQuery(t *testing.T) { Convey("TestBuildTracesQuery", t, func() { query, err := buildTracesQuery(tt.Start, tt.End, tt.Step, tt.BuilderQuery, tt.TableName, map[string]v3.AttributeKey{ "name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, - }, tt.PanelType) + }, tt.PanelType, "") + So(err, ShouldBeNil) + So(query, ShouldEqual, tt.ExpectedQuery) + }) + } +} + +var testPrepTracesQueryData = []struct { + Name string + PanelType v3.PanelType + Start int64 + End int64 + BuilderQuery *v3.BuilderQuery + ExpectedQuery string + Keys map[string]v3.AttributeKey + Type string +}{ + { + Name: "Test TS with limit- first", + PanelType: v3.PanelTypeGraph, + Start: 1680066360726210000, + End: 1680066458000000000, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + Limit: 10, + StepInterval: 60, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + }, + ExpectedQuery: "SELECT `method` from (SELECT stringTagMap['method'] as `method`," + + " toFloat64(count(distinct(stringTagMap['name']))) as value from signoz_traces.distributed_signoz_index_v2" + + " where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND" + + " stringTagMap['method'] = 'GET' AND has(stringTagMap, 'method') group by `method` order by value DESC) LIMIT 10", + Keys: map[string]v3.AttributeKey{"name": {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, + Type: constants.FirstQueryGraphLimit, + }, + { + Name: "Test TS with limit- first - with order by value", + PanelType: v3.PanelTypeGraph, + Start: 1680066360726210000, + End: 1680066458000000000, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + Limit: 10, + StepInterval: 60, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: constants.SigNozOrderByValue, Order: "ASC"}}, + }, + ExpectedQuery: "SELECT `method` from (SELECT stringTagMap['method'] as `method`," + + " toFloat64(count(distinct(stringTagMap['name']))) as value from " + + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + + " AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" + + " has(stringTagMap, 'method') group by `method` order by value ASC) LIMIT 10", + Keys: map[string]v3.AttributeKey{}, + Type: constants.FirstQueryGraphLimit, + }, + { + Name: "Test TS with limit- first - with order by attribute", + PanelType: v3.PanelTypeGraph, + Start: 1680066360726210000, + End: 1680066458000000000, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{}, + Limit: 10, + StepInterval: 60, + GroupBy: []v3.AttributeKey{{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}}, + OrderBy: []v3.OrderBy{{ColumnName: "serviceName", Order: "ASC"}}, + }, + ExpectedQuery: "SELECT `serviceName` from (SELECT serviceName as `serviceName`," + + " toFloat64(count(distinct(serviceName))) as value from " + + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + + " AND timestamp <= '1680066458000000000') " + + "group by `serviceName` order by `serviceName` ASC) LIMIT 10", + Keys: map[string]v3.AttributeKey{}, + Type: constants.FirstQueryGraphLimit, + }, + { + Name: "Test TS with limit- first - with 2 group by and 2 order by", + PanelType: v3.PanelTypeGraph, + Start: 1680066360726210000, + End: 1680066458000000000, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{}, + Limit: 10, + StepInterval: 60, + GroupBy: []v3.AttributeKey{ + {Key: "serviceName", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, + {Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + }, + OrderBy: []v3.OrderBy{{ColumnName: "serviceName", Order: "ASC"}, {ColumnName: constants.SigNozOrderByValue, Order: "ASC"}}, + }, + ExpectedQuery: "SELECT `serviceName`,`http.method` from (SELECT serviceName as `serviceName`," + + " stringTagMap['http.method'] as `http.method`," + + " toFloat64(count(distinct(serviceName))) as value from " + + "signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + + " AND timestamp <= '1680066458000000000') AND has(stringTagMap, 'http.method') " + + "group by `serviceName`,`http.method` order by `serviceName` ASC,value ASC) LIMIT 10", + Keys: map[string]v3.AttributeKey{}, + Type: constants.FirstQueryGraphLimit, + }, + { + Name: "Test TS with limit- second", + PanelType: v3.PanelTypeGraph, + Start: 1680066360726210000, + End: 1680066458000000000, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + Limit: 2, + StepInterval: 60, + }, + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " + + "stringTagMap['method'] as `method`, toFloat64(count(distinct(stringTagMap['name'])))" + + " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + + " AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" + + " has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by value DESC", + Keys: map[string]v3.AttributeKey{}, + Type: constants.SecondQueryGraphLimit, + }, + { + Name: "Test TS with limit- second - with order by", + PanelType: v3.PanelTypeGraph, + Start: 1680066360726210000, + End: 1680066458000000000, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + GroupBy: []v3.AttributeKey{{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}}, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}}, + Limit: 2, + StepInterval: 60, + }, + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " + + "stringTagMap['method'] as `method`, toFloat64(count(distinct(stringTagMap['name'])))" + + " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + + " AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" + + " has(stringTagMap, 'method') AND (`method`) GLOBAL IN (%s) group by `method`,ts order by `method` ASC", Keys: map[string]v3.AttributeKey{}, + Type: constants.SecondQueryGraphLimit, + }, + { + Name: "Test TS with limit - second - with two group by and two order by", + PanelType: v3.PanelTypeGraph, + Start: 1680066360726210000, + End: 1680066458000000000, + BuilderQuery: &v3.BuilderQuery{ + QueryName: "A", + AggregateAttribute: v3.AttributeKey{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + AggregateOperator: v3.AggregateOperatorCountDistinct, + Expression: "A", + Filters: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{ + {Key: v3.AttributeKey{Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "GET", Operator: "="}, + }, + }, + GroupBy: []v3.AttributeKey{ + {Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + {Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, + }, + OrderBy: []v3.OrderBy{{ColumnName: "method", Order: "ASC"}, {ColumnName: "name", Order: "ASC"}}, + Limit: 2, + StepInterval: 60, + }, + ExpectedQuery: "SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, " + + "stringTagMap['method'] as `method`, stringTagMap['name'] as `name`," + + " toFloat64(count(distinct(stringTagMap['name'])))" + + " as value from signoz_traces.distributed_signoz_index_v2 where (timestamp >= '1680066360726210000'" + + " AND timestamp <= '1680066458000000000') AND stringTagMap['method'] = 'GET' AND" + + " has(stringTagMap, 'method') AND has(stringTagMap, 'name') " + + "AND (`method`,`name`) GLOBAL IN (%s) group by `method`,`name`,ts " + + "order by `method` ASC,`name` ASC", + Keys: map[string]v3.AttributeKey{}, + Type: constants.SecondQueryGraphLimit, + }, +} + +func TestPrepareTracesQuery(t *testing.T) { + for _, tt := range testPrepTracesQueryData { + Convey("TestPrepareTracesQuery", t, func() { + query, err := PrepareTracesQuery(tt.Start, tt.End, tt.PanelType, tt.BuilderQuery, tt.Keys, tt.Type) So(err, ShouldBeNil) So(query, ShouldEqual, tt.ExpectedQuery) })